hermes_state_runtime.py 56 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005
  1. """Canonical runtime transactions; callers authorize and resolve session identity.
  2. No execution, transport, or authority objects live here. Every mutation uses the
  3. SessionDB transaction owner, including its inode guard and SQLite retry policy.
  4. """
  5. import json
  6. import uuid
  7. from typing import NamedTuple, Optional
  8. from agent.conversation_compression_archive import ABSORBED_ROW_IDS
  9. from agent.message_metadata import (
  10. CANONICAL_ROW, DATA_USE, DB_ROW_SNAPSHOT, MESSAGE_UID, PERSISTENCE_ONLY_MESSAGE_FIELDS, TOOL_CALL_UID, TOOL_CALL_UIDS,
  11. )
  12. from hermes_state_keys import admission_fingerprint
  13. class SessionGoneEvent(NamedTuple):
  14. """The ``on_session_delete`` event a committed runtime mutation owes its plugins."""
  15. session_ids: tuple
  16. reason: str
  17. class RuntimeStoreError(ValueError):
  18. """A refusal by the state authority. ``reason`` is the bounded wire code (callers compare it as a
  19. string); ``field`` and ``detail`` name the check that fired and why, so a client and the gateway
  20. log can tell the reasons' many checks apart instead of reporting the bare code."""
  21. def __init__(self, reason: str, *, field: str | None = None, detail: str | None = None):
  22. self.reason = reason
  23. self.field = _one_line(field, 80)
  24. self.detail = _one_line(detail, 160)
  25. super().__init__(reason)
  26. #: The one ``admission_conflict`` that proves the request was already admitted: this principal's
  27. #: request id on this target holds an admission with another payload. Every other conflict (route,
  28. #: binding, lineage) carries no such proof, so a client tells them apart by this field.
  29. ADMITTED_REQUEST_CONFLICT = {'field': 'request_id', 'detail': 'this request id was already admitted with another payload'}
  30. def _one_line(value, limit):
  31. """A refusal's ``field``/``detail`` as the log line and the client frame carry it: either may hold
  32. client text, so whitespace collapses to single spaces, other non-printables drop (no forged log
  33. lines), and the length is bounded below the client's frame limit."""
  34. if value is None:
  35. return None
  36. text = ''.join(ch for ch in ' '.join(str(value).split()) if ch.isprintable())
  37. return text[:limit] or None
  38. def _text(value):
  39. if not isinstance(value, str) or not value or len(value) > 1024:
  40. raise RuntimeStoreError('invalid_params')
  41. return value
  42. def _json(value):
  43. if not isinstance(value, dict):
  44. raise RuntimeStoreError('invalid_params')
  45. try:
  46. return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':'), allow_nan=False)
  47. except (TypeError, ValueError, UnicodeError) as exc:
  48. raise RuntimeStoreError('invalid_params') from exc
  49. def _epoch(conn, epoch):
  50. row = conn.execute('SELECT epoch FROM runtime_epoch WHERE singleton=1').fetchone()
  51. if type(epoch) is not int or row is None or row[0] != epoch:
  52. raise RuntimeStoreError('stale_epoch')
  53. def _session(conn, session_id):
  54. row = conn.execute('SELECT * FROM sessions WHERE id=?', (session_id,)).fetchone()
  55. if row is None:
  56. raise RuntimeStoreError('not_found')
  57. return row
  58. def _admission(conn, admission_id):
  59. row = conn.execute('SELECT * FROM session_admissions WHERE admission_id=?', (admission_id,)).fetchone()
  60. if row is None:
  61. from hermes_state_terminal import terminal_admission
  62. row = terminal_admission(conn, admission_id)
  63. if row is None:
  64. raise RuntimeStoreError('not_found')
  65. return row
  66. def _retire_admission_workers(conn, row, owner_epoch):
  67. """Retire the worker rows this admission's generation owns on BOTH the logical
  68. FIFO owner and its current physical transcript (reset/compression move the private
  69. worker assignment to the child while target_session_id stays the logical ID)."""
  70. lineage = json.loads(row['lineage_json'])
  71. targets = list({row['target_session_id'], lineage[-1] if lineage else row['target_session_id']})
  72. scope = f"session_id IN ({','.join('?' * len(targets))}) AND generation=? AND owner_epoch=?"
  73. # Settlement closes an execution that never wrote execution.finish: its read copies go too.
  74. from hermes_state_runtime_workers import compact_terminal_receipts
  75. for (execution_id,) in conn.execute(f"SELECT execution_id FROM worker_executions WHERE {scope} "
  76. "AND status!='terminal'", (*targets, row['generation'], owner_epoch)).fetchall():
  77. compact_terminal_receipts(conn, execution_id)
  78. conn.execute(f"UPDATE worker_executions SET status='terminal' WHERE {scope}",
  79. (*targets, row['generation'], owner_epoch))
  80. def _row(row):
  81. if row is None:
  82. return None
  83. result = dict(row)
  84. result['payload'] = json.loads(result.pop('payload_json'))
  85. result.pop('payload_digest')
  86. result.pop('lineage_json')
  87. result.pop('retired_at', None) # tombstone retention stamp, not admission state
  88. return result
  89. def begin_runtime_epoch(db, *, instance_id: str) -> int:
  90. """Caller MUST hold the profile ownership lock, not merely an endpoint ticket."""
  91. _text(instance_id)
  92. def write(conn):
  93. conn.execute('''INSERT INTO runtime_epoch(singleton,epoch,instance_id) VALUES(1,1,?)
  94. ON CONFLICT(singleton) DO UPDATE SET epoch=runtime_epoch.epoch+1, instance_id=excluded.instance_id''', (instance_id,))
  95. return conn.execute('SELECT epoch FROM runtime_epoch WHERE singleton=1').fetchone()[0]
  96. return db._execute_write(write)
  97. # Main's busy-queue cap (GatewayRunner._BUSY_QUEUE_MAX_PENDING): queued inputs one session holds.
  98. ADMISSION_QUEUE_MAX = 32
  99. def admit_session_input(db, *, epoch: int, principal_id: str, session_id: str,
  100. request_id: str, payload: dict, intent: str = 'queue',
  101. _authorize_write=None, _replay_guard=None) -> dict:
  102. """Admit input; the trusted private guard raises to refuse a NEW write.
  103. The guard receives the owning transaction connection, must not commit it or
  104. perform external effects, and may run again on SQLite retry. Exact existing
  105. and terminal replays bypass it: they cannot create or change accepted work.
  106. ``_replay_guard(conn)`` runs on the exact live replay instead, inside the same
  107. transaction (a replacement's binding to the admission it retired).
  108. """
  109. for value in (principal_id, session_id, request_id):
  110. _text(value)
  111. if intent not in ('queue', 'steer', 'redirect'):
  112. raise RuntimeStoreError('invalid_params')
  113. encoded = _json(payload)
  114. from hermes_state_terminal import retry_terminal_admission
  115. retired = retry_terminal_admission(db, epoch=epoch, principal_id=principal_id, session_id=session_id,
  116. request_id=request_id, payload=payload, intent=intent)
  117. if retired is not None:
  118. return retired
  119. digest = admission_fingerprint(canonical_target=session_id, payload={'input': json.loads(encoded), 'intent': intent})
  120. def write(conn):
  121. _epoch(conn, epoch)
  122. _session(conn, session_id)
  123. old = conn.execute('''SELECT * FROM session_admissions
  124. WHERE principal_id=? AND target_session_id=? AND request_id=?''', (principal_id, session_id, request_id)).fetchone()
  125. if old is not None:
  126. if old['payload_digest'] != digest:
  127. raise RuntimeStoreError('admission_conflict', **ADMITTED_REQUEST_CONFLICT)
  128. if _replay_guard is not None:
  129. _replay_guard(conn)
  130. return _row(old)
  131. queued = conn.execute('''SELECT COUNT(*) FROM session_admissions WHERE target_session_id=?
  132. AND status='queued' ''', (session_id,)).fetchone()[0]
  133. # Main's cap holds in both states: a FIFO paused behind an unknown turn runs every accepted
  134. # input after Discard, so it must not grow without bound while nobody resolves it.
  135. if queued >= ADMISSION_QUEUE_MAX:
  136. raise RuntimeStoreError('queue_full')
  137. if _authorize_write is not None:
  138. _authorize_write(conn)
  139. admission_id = uuid.uuid4().hex
  140. conn.execute('''INSERT INTO session_admissions(admission_id,request_id,principal_id,
  141. target_session_id,lineage_json,payload_json,payload_digest,intent,status,owner_epoch)
  142. VALUES(?,?,?,?,?,?,?,?,'queued',?)''',
  143. (admission_id, request_id, principal_id, session_id, json.dumps([session_id]), encoded, digest, intent, epoch))
  144. return _row(_admission(conn, admission_id))
  145. return db._execute_write(write)
  146. def get_session_admission(db, *, admission_id: str) -> dict | None:
  147. with db._read_ctx() as conn:
  148. from hermes_state_terminal import terminal_admission
  149. row = conn.execute('SELECT * FROM session_admissions WHERE admission_id=?', (admission_id,)).fetchone()
  150. return _row(row if row is not None else terminal_admission(conn, admission_id))
  151. def find_session_admission(db, *, principal_id: str, session_id: str, request_id: str) -> dict | None:
  152. """The admission at its exact durable identity: the UNIQUE (principal, target, request) key."""
  153. with db._read_ctx() as conn:
  154. return _row(conn.execute('SELECT * FROM session_admissions WHERE principal_id=? AND target_session_id=? '
  155. 'AND request_id=?', (principal_id, session_id, request_id)).fetchone())
  156. def list_session_admissions(db, *, session_id: str, pending_only: bool = True) -> list[dict]:
  157. with db._read_ctx() as conn:
  158. # Pending is an explicit index-range IN, not ``!='terminal'``: a negated status cannot
  159. # bound the (target, status, seq) index, so every probe scanned the session's history.
  160. sql = ("SELECT * FROM session_admissions WHERE target_session_id=? "
  161. + ("AND status IN ('queued','started','unknown') " if pending_only else "") + "ORDER BY seq")
  162. return [_row(row) for row in conn.execute(sql, (session_id,))]
  163. _ANY_HEAD = object()
  164. def claim_session_input(db, *, epoch: int, session_id: str, _guard=None, head=_ANY_HEAD) -> dict | bool | None:
  165. """Claim the FIFO head. ``_guard(conn)`` (owner admission gate) raises to refuse inside the
  166. write transaction, so a drain that began while this waited on the writer still wins.
  167. ``head`` is the admission id the caller's preclaim checks validated (None: it saw nothing
  168. queued); if the head is another row by the time this write runs, nothing is claimed and False
  169. tells the caller to re-read and validate the row that is now first."""
  170. def write(conn):
  171. _epoch(conn, epoch)
  172. if _guard is not None:
  173. _guard(conn)
  174. session = _session(conn, session_id)
  175. blocked = conn.execute("SELECT status FROM session_admissions WHERE target_session_id=? AND status IN ('started','unknown')", (session_id,)).fetchall()
  176. if any(row[0] == 'unknown' for row in blocked):
  177. raise RuntimeStoreError('unknown_execution')
  178. from hermes_state_runtime_workers import worker_states
  179. workers = worker_states(conn, session_id)
  180. if 'unknown' in workers:
  181. raise RuntimeStoreError('unknown_execution')
  182. if blocked or workers:
  183. return None
  184. row = conn.execute("SELECT * FROM session_admissions WHERE target_session_id=? AND status='queued' ORDER BY seq LIMIT 1", (session_id,)).fetchone()
  185. if head is not _ANY_HEAD and (row['admission_id'] if row is not None else None) != head:
  186. return False
  187. if row is None:
  188. return None
  189. generation = session['runtime_generation'] + 1
  190. conn.execute('UPDATE sessions SET runtime_generation=? WHERE id=?', (generation, session_id))
  191. changed = conn.execute("UPDATE session_admissions SET status='started',owner_epoch=?,generation=? WHERE admission_id=? AND status='queued'", (epoch, generation, row['admission_id']))
  192. if changed.rowcount != 1:
  193. raise RuntimeStoreError('stale_generation')
  194. claimed = _row(_admission(conn, row['admission_id']))
  195. # The transcript high-water mark at the claim (``messages.id`` is AUTOINCREMENT across every
  196. # session): a row above it on this conversation was written by this turn, so settlement can
  197. # tell this turn's own failed-turn boundary from an earlier turn's (#134889 GW-21).
  198. claimed['transcript_mark'] = conn.execute('SELECT COALESCE(MAX(id), 0) FROM messages').fetchone()[0]
  199. return claimed
  200. return db._execute_write(write)
  201. def settle_session_input(db, *, epoch: int, admission_id: str, generation: int, outcome: str,
  202. result: dict | None = None, _transfers=None, _terminal_write=None) -> dict:
  203. """``_terminal_write(conn, row)`` is the owner's same-transaction effect (a transcript notice,
  204. ``gateway.session_results.finish_result``); like ``resolve_unknown_session_input``'s it must
  205. not commit, may run again on SQLite retry, and raising rolls the settlement back."""
  206. if outcome not in ('completed', 'interrupted', 'rejected', 'failed'):
  207. raise RuntimeStoreError('invalid_params')
  208. from hermes_state_terminal import compact_result, encode_result
  209. def write(conn):
  210. _epoch(conn, epoch)
  211. row = _admission(conn, admission_id)
  212. session = _session(conn, row['target_session_id'])
  213. if (row['status'] != 'started' or row['owner_epoch'] != epoch
  214. or type(generation) is not int or row['generation'] != generation
  215. or session['runtime_generation'] != generation):
  216. raise RuntimeStoreError('stale_generation')
  217. _retire_admission_workers(conn, row, epoch)
  218. if _terminal_write is not None:
  219. _terminal_write(conn, row)
  220. if result is not None:
  221. from hermes_state_terminal import RESULT_PREFIX
  222. encoded = _json(encode_result(compact_result(result, user_message=json.loads(row['payload_json']).get('text'))))
  223. conn.execute('INSERT INTO state_meta(key,value) VALUES(?,?) '
  224. 'ON CONFLICT(key) DO UPDATE SET value=excluded.value',
  225. (RESULT_PREFIX + admission_id, encoded))
  226. conn.execute("UPDATE session_admissions SET status='terminal',outcome=? WHERE admission_id=?", (outcome, admission_id))
  227. settle_steer_transfers(conn, epoch=epoch, generation=generation, transfers=_transfers, outcome=outcome)
  228. conn.execute('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?', (row['target_session_id'],))
  229. return _row(_admission(conn, admission_id))
  230. return db._execute_write(write)
  231. _CANCEL_QUEUED = "UPDATE session_admissions SET status='terminal',outcome='cancelled' WHERE admission_id=?"
  232. def cancel_session_input(db, *, epoch: int, admission_id: str) -> dict:
  233. def write(conn):
  234. _epoch(conn, epoch)
  235. row = _admission(conn, admission_id)
  236. if row['status'] == 'unknown':
  237. raise RuntimeStoreError('unknown_execution')
  238. # A steer in transfer reads as its running claim and settles with it: as started, not cancellable.
  239. if row['status'] == 'started' or row['outcome'] == STEERING:
  240. raise RuntimeStoreError('stale_generation')
  241. if row['status'] == 'queued':
  242. conn.execute(_CANCEL_QUEUED, (admission_id,))
  243. return _row(_admission(conn, admission_id))
  244. return db._execute_write(write)
  245. # A queued admission handed to its FIFO's running claim: terminal (never claimable, outside the queue
  246. # cap) but not yet an outcome. It is bound to that claim's ``(owner_epoch, generation)`` and settles
  247. # with it: ``steered`` once the claim completes with the text delivered, ``interrupted`` otherwise, or
  248. # back to ``queued`` (its own FIFO place) when the text never reached the claim's agent.
  249. STEERING = 'steering'
  250. def steer_replayed(row) -> bool:
  251. """``row`` is a committed transfer (still in flight, or settled ``steered``): an exact retry of
  252. ``prompt.steer_queued`` replays it."""
  253. return row['status'] == 'terminal' and row['outcome'] in (STEERING, 'steered')
  254. def _requeue_transfer(conn, *, admission_id: str, epoch: int, generation: int) -> bool:
  255. """A transfer still bound to the claim ``(epoch, generation)`` goes back to its FIFO place (``seq``)."""
  256. return conn.execute("UPDATE session_admissions SET status='queued',outcome=NULL,generation=NULL "
  257. "WHERE admission_id=? AND status='terminal' AND outcome=? AND owner_epoch=? "
  258. "AND generation=?", (admission_id, STEERING, epoch, generation)).rowcount == 1
  259. def steer_session_input(db, *, epoch: int, admission_id: str, generation: int) -> tuple[dict, bool]:
  260. """Hand a still-queued admission to its FIFO's running claim: queued -> terminal ``steering``
  261. bound to ``generation`` in one write, only while ``generation`` is the started claim this owner
  262. holds, so the drain can never also claim it. ``(row, True)`` when this call committed the
  263. transfer, ``(row, False)`` for an exact retry of one already committed (never delivered twice)."""
  264. def write(conn):
  265. _epoch(conn, epoch)
  266. row = _admission(conn, admission_id)
  267. if steer_replayed(row):
  268. return _row(row), False
  269. if row['status'] != 'queued':
  270. raise RuntimeStoreError('admission_started')
  271. if type(generation) is not int or conn.execute(
  272. "SELECT 1 FROM session_admissions WHERE target_session_id=? AND status='started' "
  273. "AND owner_epoch=? AND generation=?", (row['target_session_id'], epoch, generation)).fetchone() is None:
  274. raise RuntimeStoreError('session_idle')
  275. conn.execute("UPDATE session_admissions SET status='terminal',outcome=?,owner_epoch=?,generation=? "
  276. "WHERE admission_id=?", (STEERING, epoch, generation, admission_id))
  277. return _row(_admission(conn, admission_id)), True
  278. return db._execute_write(write)
  279. def settle_steer_transfers(conn, *, epoch: int, generation: int, transfers, outcome: str) -> None:
  280. """In the receiving claim's terminal write: each admission bound to it settles ``steered`` only if
  281. its text reached the claim's agent (``transfers[id]``) and the claim completed; a delivered text
  282. under any other ending is ``interrupted`` (never replayed); an undelivered one is queued again."""
  283. for admission_id, delivered in (transfers or {}).items():
  284. if not delivered:
  285. _requeue_transfer(conn, admission_id=admission_id, epoch=epoch, generation=generation)
  286. continue
  287. conn.execute("UPDATE session_admissions SET outcome=? WHERE admission_id=? AND status='terminal' "
  288. "AND outcome=? AND owner_epoch=? AND generation=?",
  289. ('steered' if outcome == 'completed' else 'interrupted', admission_id, STEERING, epoch, generation))
  290. def revert_steer_input(db, *, epoch: int, admission_id: str, generation: int) -> bool:
  291. """A transfer whose text never reached the claim's agent: the admission is queued again, in its
  292. original FIFO place (its ``seq``). False when the claim's settlement already decided it."""
  293. def write(conn):
  294. _epoch(conn, epoch)
  295. return _requeue_transfer(conn, admission_id=admission_id, epoch=epoch, generation=generation)
  296. return db._execute_write(write)
  297. REPLACEMENT_PREFIX = 'gateway.queued_replacement.v1.'
  298. def replacement_binding(principal_id: str, session_id: str, request_id: str) -> str:
  299. """The durable key binding a replacement's identity to the admission it retired."""
  300. return REPLACEMENT_PREFIX + admission_fingerprint(canonical_target=session_id,
  301. payload={'principal': principal_id, 'request': request_id})
  302. def retire_replaced_input(conn, admission_id: str, binding: str) -> None:
  303. """Inside the replacement's own admission write: the replaced row, still queued, is cancelled
  304. in the same transaction that admits its successor at the FIFO tail, which is bound to it."""
  305. row = _admission(conn, admission_id)
  306. if row['status'] != 'queued':
  307. raise RuntimeStoreError('admission_started')
  308. conn.execute(_CANCEL_QUEUED, (admission_id,))
  309. conn.execute('INSERT OR REPLACE INTO state_meta(key,value) VALUES(?,?)', (binding, json.dumps(admission_id)))
  310. def check_replaced_input(conn, admission_id: str, binding: str) -> None:
  311. """The exact replay of a replacement answers only the request that retired ``admission_id``."""
  312. saved = conn.execute('SELECT value FROM state_meta WHERE key=?', (binding,)).fetchone()
  313. if saved is None or json.loads(saved[0]) != admission_id:
  314. raise RuntimeStoreError('admission_conflict')
  315. def rehome_queued_inputs(db, *, epoch: int, old_session_id: str, new_session_id: str,
  316. principal_prefix: str) -> list[str]:
  317. """Re-target the old owner's still-queued ``principal_prefix*`` admissions onto
  318. ``new_session_id`` in one transaction (the compression-exhaustion auto-reset). Each row keeps
  319. its admission id, request id and seq, so FIFO order and retry dedupe survive; the digest is
  320. re-keyed to the new target. Nothing moves while the old FIFO holds an ``unknown`` row (the
  321. operator's to resume). Returns the moved admission ids in FIFO order."""
  322. for value in (old_session_id, new_session_id, principal_prefix):
  323. _text(value)
  324. def write(conn):
  325. _epoch(conn, epoch)
  326. _session(conn, new_session_id)
  327. if conn.execute("SELECT 1 FROM session_admissions WHERE target_session_id=? AND status='unknown'",
  328. (old_session_id,)).fetchone():
  329. return []
  330. rows = conn.execute("SELECT * FROM session_admissions WHERE target_session_id=? AND status='queued' "
  331. "AND substr(principal_id,1,?)=? ORDER BY seq",
  332. (old_session_id, len(principal_prefix), principal_prefix)).fetchall()
  333. for row in rows:
  334. digest = admission_fingerprint(canonical_target=new_session_id, payload={
  335. 'input': json.loads(row['payload_json']), 'intent': row['intent']})
  336. conn.execute("UPDATE session_admissions SET target_session_id=?,lineage_json=?,payload_digest=? "
  337. "WHERE admission_id=? AND status='queued'",
  338. (new_session_id, json.dumps([new_session_id]), digest, row['admission_id']))
  339. if rows:
  340. for sid in (old_session_id, new_session_id):
  341. conn.execute('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?', (sid,))
  342. return [row['admission_id'] for row in rows]
  343. return db._execute_write(write)
  344. def retarget_local_automation(db, *, epoch: int, session_id: str, principal_id: str,
  345. old_targets, new_target: str) -> list[str]:
  346. """Point a LOCAL owner's still-queued ``principal_id`` automation rows (a Bot Chat's queued
  347. ``hermes peer dm`` turns, plugin and completion notices) that name a pre-reset transcript in
  348. ``old_targets`` at ``new_target``, in one transaction (the compression-exhaustion auto-reset).
  349. A LOCAL FIFO's owner is its receipt id and survives the reset; only the descriptor's ``target``
  350. goes stale, and the drain refuses a stale one (``admission_conflict``), pausing the FIFO for
  351. good. Admission id, request id and seq are kept; the digest is re-keyed to the new payload.
  352. Nothing moves behind an ``unknown`` head. Returns the retargeted admission ids in FIFO order."""
  353. for value in (session_id, principal_id, new_target, *old_targets):
  354. _text(value)
  355. old_targets = set(old_targets)
  356. def write(conn):
  357. _epoch(conn, epoch)
  358. _session(conn, new_target)
  359. if conn.execute("SELECT 1 FROM session_admissions WHERE target_session_id=? AND status='unknown'",
  360. (session_id,)).fetchone():
  361. return []
  362. moved = []
  363. for row in conn.execute("SELECT * FROM session_admissions WHERE target_session_id=? AND status='queued' "
  364. "AND principal_id=? ORDER BY seq", (session_id, principal_id)).fetchall():
  365. payload = json.loads(row['payload_json'])
  366. descriptor = payload.get('local_automation_v1') if isinstance(payload, dict) else None
  367. if not isinstance(descriptor, dict) or descriptor.get('target') not in old_targets:
  368. continue
  369. descriptor['target'] = new_target
  370. digest = admission_fingerprint(canonical_target=session_id, payload={'input': payload, 'intent': row['intent']})
  371. conn.execute("UPDATE session_admissions SET payload_json=?,payload_digest=? WHERE admission_id=? AND status='queued'",
  372. (_json(payload), digest, row['admission_id']))
  373. moved.append(row['admission_id'])
  374. if moved:
  375. conn.execute('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?', (session_id,))
  376. return moved
  377. return db._execute_write(write)
  378. def recover_session_inputs(db, *, epoch: int) -> int:
  379. """Never replay started work. Live worker adoption is a separate explicit operation."""
  380. def write(conn):
  381. _epoch(conn, epoch)
  382. from hermes_state_runtime_workers import retire_stale_children
  383. retire_stale_children(conn, epoch)
  384. conn.execute("UPDATE worker_executions SET status='unknown' WHERE status IN ('registered','running') AND owner_epoch!=?", (epoch,))
  385. # A steer handed to a lost claim may or may not have reached its agent: never replayed.
  386. conn.execute("UPDATE session_admissions SET outcome='interrupted' WHERE status='terminal' AND outcome=? "
  387. "AND owner_epoch!=?", (STEERING, epoch))
  388. return conn.execute("UPDATE session_admissions SET status='unknown' WHERE status='started' AND owner_epoch!=?", (epoch,)).rowcount
  389. return db._execute_write(write)
  390. def mutation_receipt_key(principal_id, session_id, request_id, *, operation, payload, expected_revision,
  391. expected_generation=None):
  392. """``(state_meta key, digest)`` of a mutation request's durable exact-retry receipt."""
  393. key = 'gateway.mutation.v1.' + admission_fingerprint(
  394. canonical_target=session_id, payload={'principal': principal_id, 'request': request_id})
  395. fingerprint = {'operation': operation, 'payload': payload, 'expected_revision': expected_revision}
  396. if expected_generation is not None:
  397. fingerprint['expected_generation'] = expected_generation
  398. return key, admission_fingerprint(canonical_target=session_id, payload=fingerprint)
  399. def replay_mutation_receipt(conn, key, digest):
  400. """The stored result of an exact retry, ``None`` when unseen; a different request refuses."""
  401. old = conn.execute('SELECT value FROM state_meta WHERE key=?', (key,)).fetchone()
  402. if old is None:
  403. return None
  404. receipt = json.loads(old[0])
  405. if receipt['digest'] != digest:
  406. raise RuntimeStoreError('admission_conflict')
  407. return receipt['result']
  408. def mutate_runtime_session(db, *, epoch: int, principal_id: str, session_id: str,
  409. request_id: str, expected_revision: int,
  410. operation: str, payload: dict, expected_generation: int | None = None,
  411. _live_guard=None, _authorize_write=None, _prepare_only=False, _prepared=None) -> dict:
  412. """Commit a closed metadata edit and its retry receipt in the same transaction.
  413. Caller authorizes the principal and resolves the canonical session. These
  414. metadata edits do not stop execution. Existing direct writers must migrate
  415. before this seam can provide universal revision fencing.
  416. """
  417. for value in (principal_id, session_id, request_id):
  418. _text(value)
  419. if type(expected_revision) is not int or expected_revision < 0:
  420. raise RuntimeStoreError('invalid_params')
  421. from hermes_state_mutations import validate_action, apply_action
  422. validate_action(operation, payload)
  423. if expected_generation is not None and (type(expected_generation) is not int or expected_generation < 0):
  424. raise RuntimeStoreError('invalid_params')
  425. if operation in {'delete', 'rewind', 'reset', 'branch', 'model', 'compress'} and expected_generation is None:
  426. raise RuntimeStoreError('invalid_params')
  427. # Snapshot caller data before waiting for the writer lock.
  428. payload = json.loads(_json(payload))
  429. key, digest = mutation_receipt_key(principal_id, session_id, request_id, operation=operation, payload=payload,
  430. expected_revision=expected_revision, expected_generation=expected_generation)
  431. def write(conn) -> tuple[dict, Optional[SessionGoneEvent]]:
  432. """The receipt result plus the on_session_delete event this commit owes (None on replay)."""
  433. _epoch(conn, epoch)
  434. replay = replay_mutation_receipt(conn, key, digest)
  435. if replay is not None:
  436. return replay, None
  437. session = conn.execute('SELECT * FROM sessions WHERE id=?', (session_id,)).fetchone()
  438. if session is None and operation != 'import':
  439. raise RuntimeStoreError('not_found')
  440. revision = session['runtime_revision'] if session else 0
  441. if revision != expected_revision:
  442. raise RuntimeStoreError('revision_conflict')
  443. if expected_generation is not None and (session is None or session['runtime_generation'] != expected_generation):
  444. raise RuntimeStoreError('stale_generation')
  445. if _live_guard is not None:
  446. from hermes_state_mutation_guards import delete_targets
  447. targets = delete_targets(conn, session_id) if operation == 'delete' else [session_id]
  448. _live_guard(targets)
  449. if _authorize_write is not None:
  450. _authorize_write(conn)
  451. if _prepare_only:
  452. from hermes_state_mutation_prepared import local_snapshot
  453. snapshot = local_snapshot(db, conn, session_id)
  454. if operation == 'branch':
  455. # The branch commit's own idle rule (queued input too), before the owner starts servers.
  456. from hermes_state_mutation_guards import require_idle
  457. require_idle(db, conn, list({session_id, snapshot['target']}))
  458. return {'snapshot': snapshot}, None
  459. affected, projection = apply_action(db, conn, session_id, operation, payload, prepared=_prepared)
  460. event = None
  461. if operation == 'delete':
  462. event = SessionGoneEvent(tuple(projection.get('deleted_ids') or (session_id,)), 'delete')
  463. elif payload.get('archived') is True and session is not None and not session['archived']:
  464. event = SessionGoneEvent(tuple(affected) or (session_id,), 'archive')
  465. conn.executemany('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?',
  466. [(target,) for target in affected])
  467. updated = conn.execute('SELECT * FROM sessions WHERE id=?', (session_id,)).fetchone()
  468. result = {'session_id': session_id, 'revision': updated['runtime_revision'] if updated else expected_revision + 1,
  469. 'execution_generation': updated['runtime_generation'] if updated else expected_generation,
  470. 'operation': operation, **projection}
  471. conn.execute('INSERT INTO state_meta(key,value) VALUES(?,?)',
  472. (key, _json({'digest': digest, 'result': result})))
  473. return result, event
  474. result, event = db._execute_write(write)
  475. if event is not None:
  476. from hermes_state_common import emit_session_deleted
  477. emit_session_deleted(db, event.session_ids, reason=event.reason, primary=session_id)
  478. return result
  479. _IMPORT_KEY = 'gateway.prompt_admissions_import.v1'
  480. def _canonical_chain(conn, session_id):
  481. # Same selector as SessionDB.get_compression_chain, on OUR transaction connection.
  482. from hermes_state_compression import _CHAIN_STEP_SQL
  483. _session(conn, session_id)
  484. chain = [session_id]
  485. for _ in range(100):
  486. child = conn.execute(_CHAIN_STEP_SQL, (chain[-1],)).fetchone()
  487. if child is None:
  488. return chain
  489. if child['id'] in chain:
  490. raise RuntimeStoreError('admission_conflict')
  491. chain.append(child['id'])
  492. raise RuntimeStoreError('admission_conflict')
  493. def _import_legacy_row(conn, row, epoch, principal_id):
  494. for field in ('admission_id', 'target_session_id', 'root'):
  495. _text(row[field])
  496. lineage = json.loads(row['lineage'])
  497. if not isinstance(lineage, list) or row['target_session_id'] not in lineage:
  498. raise RuntimeStoreError('invalid_params')
  499. payload = json.loads(row['payload'])
  500. encoded = _json(payload)
  501. intent = payload.get('intent', 'queue')
  502. intent = 'redirect' if intent == 'interrupt' else intent
  503. if intent not in ('queue', 'steer', 'redirect') or row['status'] not in ('queued', 'started', 'unknown', 'terminal'):
  504. raise RuntimeStoreError('invalid_params')
  505. chain = _canonical_chain(conn, row['target_session_id'])
  506. target = chain[-1]
  507. status = 'unknown' if row['status'] == 'started' else row['status']
  508. generation = row['generation']
  509. if generation is None and status == 'unknown':
  510. generation = _session(conn, target)['runtime_generation'] + 1
  511. if generation is not None and (type(generation) is not int or generation < 0):
  512. raise RuntimeStoreError('invalid_params')
  513. digest = admission_fingerprint(canonical_target=target, payload={'input': payload, 'intent': intent})
  514. conn.execute("""INSERT INTO session_admissions(admission_id,request_id,principal_id,
  515. target_session_id,lineage_json,payload_json,payload_digest,intent,status,outcome,owner_epoch,generation)
  516. VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""", (row['admission_id'], row['admission_id'], principal_id,
  517. target, json.dumps(chain), encoded, digest, intent, status, row['outcome'], epoch, generation))
  518. if generation is not None:
  519. conn.execute('UPDATE sessions SET runtime_generation=MAX(runtime_generation,?) WHERE id=?', (generation, target))
  520. def import_legacy_session_admissions(db, *, epoch: int, source_path, principal_id: str,
  521. writers_drained: bool) -> int:
  522. """Import a frozen legacy snapshot; original file remains rollback evidence.
  523. The caller proves the old writers are drained before invoking this function.
  524. Fingerprinting uses SQL snapshot data, never a raw open/close on a live inode.
  525. """
  526. from contextlib import closing
  527. from pathlib import Path
  528. import hashlib
  529. import sqlite3
  530. from hermes_cli.sqlite_safe_read import connect_tracked
  531. if writers_drained is not True:
  532. raise RuntimeStoreError('invalid_params')
  533. _text(principal_id)
  534. source_path = Path(source_path).resolve(strict=True)
  535. with closing(connect_tracked(source_path.as_uri() + '?mode=ro', uri=True)) as source:
  536. source.row_factory = sqlite3.Row
  537. source.execute('PRAGMA query_only=ON')
  538. source.execute('BEGIN')
  539. rows = [dict(r) for r in source.execute('SELECT * FROM admissions ORDER BY seq')]
  540. executions = []
  541. if source.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='executions'").fetchone():
  542. executions = [dict(r) for r in source.execute('SELECT * FROM executions ORDER BY root')]
  543. fingerprint = hashlib.sha256(_json({'rows': rows, 'executions': executions}).encode('utf-8')).hexdigest()
  544. marker = _json({'source': str(source_path), 'fingerprint': fingerprint, 'imported_id_count': len(rows), 'principal_id': principal_id})
  545. def write(conn):
  546. _epoch(conn, epoch)
  547. old = conn.execute('SELECT value FROM state_meta WHERE key=?', (_IMPORT_KEY,)).fetchone()
  548. if old is not None:
  549. if old[0] != marker:
  550. raise RuntimeStoreError('admission_conflict')
  551. return 0
  552. for execution in executions:
  553. generation = execution['generation']
  554. if type(generation) is not int or generation < 0:
  555. raise RuntimeStoreError('invalid_params')
  556. target = _canonical_chain(conn, _text(execution['root']))[-1]
  557. conn.execute('UPDATE sessions SET runtime_generation=MAX(runtime_generation,?) WHERE id=?', (generation, target))
  558. for row in rows:
  559. _import_legacy_row(conn, row, epoch, principal_id)
  560. conn.execute('INSERT INTO state_meta(key,value) VALUES(?,?)', (_IMPORT_KEY, marker))
  561. return len(rows)
  562. return db._execute_write(write)
  563. def resolve_unknown_session_input(db, *, epoch: int, admission_id: str, generation: int,
  564. _terminal_write=None, _captured=None) -> dict:
  565. """Explicit operator acknowledgement; resolves uncertainty, never requeues it.
  566. ``_terminal_write(conn, row)`` is the owner's same-transaction effect (the discarded turn's
  567. transcript boundary): it must not commit, may run again on SQLite retry, and raising rolls
  568. the whole resolution back, so the FIFO can never advance past an unclosed lost turn.
  569. ``_captured=(outcome, stored_result)`` is the exact result this owner captured for the turn
  570. before its settlement write failed; it commits with the resolution (only for an ``unknown``
  571. row stamped by this same owner epoch), so the finished answer is not thrown away."""
  572. from hermes_state_terminal import RESULT_PREFIX
  573. def write(conn):
  574. _epoch(conn, epoch)
  575. row = _admission(conn, admission_id)
  576. if (type(generation) is int and row['generation'] == generation and row['status'] == 'terminal'
  577. and (row['outcome'] == 'interrupted' or conn.execute(
  578. 'SELECT 1 FROM state_meta WHERE key=?', (RESULT_PREFIX + admission_id,)).fetchone())):
  579. return _row(row)
  580. if type(generation) is not int or row['status'] != 'unknown' or row['generation'] != generation:
  581. raise RuntimeStoreError('stale_generation')
  582. outcome = 'interrupted'
  583. if _captured is not None and row['owner_epoch'] == epoch:
  584. outcome, stored = _captured
  585. if outcome not in ('completed', 'interrupted', 'failed'):
  586. raise RuntimeStoreError('invalid_params')
  587. from hermes_state_terminal import compact_result, encode_result
  588. conn.execute('INSERT INTO state_meta(key,value) VALUES(?,?) '
  589. 'ON CONFLICT(key) DO UPDATE SET value=excluded.value',
  590. (RESULT_PREFIX + admission_id, _json(encode_result(compact_result(
  591. stored, user_message=json.loads(row['payload_json']).get('text'))))))
  592. if _terminal_write is not None:
  593. _terminal_write(conn, row)
  594. _retire_admission_workers(conn, row, row['owner_epoch'])
  595. conn.execute("UPDATE session_admissions SET status='terminal',outcome=? WHERE admission_id=?",
  596. (outcome, admission_id))
  597. conn.execute('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?', (row['target_session_id'],))
  598. return _row(_admission(conn, admission_id))
  599. return db._execute_write(write)
  600. def _worker_public(row):
  601. result = dict(row)
  602. result.pop('adoption_digest')
  603. return result
  604. def _worker_assignment(conn, execution_id, session_id, generation):
  605. row = conn.execute('SELECT * FROM worker_executions WHERE execution_id=?', (execution_id,)).fetchone()
  606. if row is None:
  607. raise RuntimeStoreError('not_found')
  608. if row['session_id'] != session_id:
  609. raise RuntimeStoreError('permission_denied')
  610. if (type(generation) is not int or row['generation'] != generation
  611. or _session(conn, session_id)['runtime_generation'] != generation):
  612. raise RuntimeStoreError('stale_generation')
  613. return row
  614. def _linked_worker_admission(conn, session_id, generation, owner_epoch, statuses):
  615. placeholders = ','.join('?' for _ in statuses)
  616. rows = conn.execute(f"""SELECT * FROM session_admissions
  617. WHERE generation=? AND owner_epoch=? AND status IN ({placeholders})
  618. AND (target_session_id=? OR EXISTS (
  619. SELECT 1 FROM json_each(session_admissions.lineage_json) WHERE value=?
  620. )) ORDER BY seq""",
  621. (generation, owner_epoch, *statuses, session_id, session_id)).fetchall()
  622. if len(rows) > 1:
  623. raise RuntimeStoreError('admission_conflict')
  624. return rows[0] if rows else None
  625. def _secret_digest(secret):
  626. import hashlib
  627. _text(secret)
  628. return hashlib.sha256(secret.encode('utf-8')).hexdigest()
  629. def register_worker_execution(db, *, epoch: int, execution_id: str, session_id: str,
  630. generation: int, kind: str, adoption_secret: str, require_idle: bool = False) -> dict:
  631. for value in (execution_id, session_id):
  632. _text(value)
  633. if kind not in ('cron', 'child', 'compute', 'kanban'):
  634. raise RuntimeStoreError('invalid_params')
  635. digest = _secret_digest(adoption_secret)
  636. def write(conn):
  637. _epoch(conn, epoch)
  638. session = _session(conn, session_id)
  639. if type(generation) is not int or session['runtime_generation'] != generation:
  640. raise RuntimeStoreError('stale_generation')
  641. old = conn.execute('SELECT * FROM worker_executions WHERE execution_id=?', (execution_id,)).fetchone()
  642. if old is not None:
  643. if (old['session_id'], old['generation'], old['kind'], old['owner_epoch'], old['adoption_digest']) != (session_id, generation, kind, epoch, digest):
  644. raise RuntimeStoreError('admission_conflict')
  645. return _worker_public(old)
  646. from hermes_state_runtime_workers import runtime_lineage, worker_states
  647. targets = runtime_lineage(conn, session_id)
  648. if require_idle and any(conn.execute("SELECT 1 FROM session_admissions WHERE target_session_id=? AND status IN ('queued','started','unknown')", (sid,)).fetchone() for sid in targets):
  649. raise RuntimeStoreError('stale_generation')
  650. if worker_states(conn, session_id):
  651. raise RuntimeStoreError('stale_generation')
  652. if any(conn.execute("SELECT 1 FROM session_admissions WHERE target_session_id=? AND status='unknown'", (sid,)).fetchone() for sid in targets):
  653. raise RuntimeStoreError('unknown_execution')
  654. conn.execute("""INSERT INTO worker_executions(execution_id,session_id,kind,owner_epoch,generation,status,adoption_digest)
  655. VALUES(?,?,?,?,?,'registered',?)""", (execution_id, session_id, kind, epoch, generation, digest))
  656. return _worker_public(_worker_assignment(conn, execution_id, session_id, generation))
  657. return db._execute_write(write)
  658. def adopt_worker_execution(db, *, epoch: int, execution_id: str, session_id: str,
  659. generation: int, adoption_secret: str) -> dict:
  660. """Authority first verifies the original producer claim and live worker proof."""
  661. import hmac
  662. digest = _secret_digest(adoption_secret)
  663. def write(conn):
  664. _epoch(conn, epoch)
  665. row = _worker_assignment(conn, execution_id, session_id, generation)
  666. if row['status'] == 'terminal':
  667. raise RuntimeStoreError('stale_generation')
  668. if not hmac.compare_digest(row['adoption_digest'], digest):
  669. raise RuntimeStoreError('permission_denied')
  670. # An admission-reserved worker reports to the reserving owner over that owner's pipes and
  671. # dies with them, so only its same-epoch handshake adopts it. A late handshake reaching a
  672. # restarted owner would re-arm the recovered `unknown` admission as `started` with nobody
  673. # left to settle it (and Discard refuses `started`); refuse it so recovery's verdict stands.
  674. if execution_id.startswith('admission-worker:') and row['owner_epoch'] != epoch:
  675. raise RuntimeStoreError('stale_epoch')
  676. linked = _linked_worker_admission(
  677. conn, session_id, generation, row['owner_epoch'], ('started', 'unknown'))
  678. if linked is not None:
  679. changed = conn.execute("""UPDATE session_admissions SET owner_epoch=?,status='started'
  680. WHERE admission_id=? AND generation=? AND owner_epoch=?
  681. AND status IN ('started','unknown')""",
  682. (epoch, linked['admission_id'], generation, row['owner_epoch']))
  683. if changed.rowcount != 1:
  684. raise RuntimeStoreError('stale_generation')
  685. changed = conn.execute(
  686. "UPDATE worker_executions SET owner_epoch=?,status='running' WHERE execution_id=?",
  687. (epoch, execution_id))
  688. if changed.rowcount != 1:
  689. raise RuntimeStoreError('stale_generation')
  690. return _worker_public(_worker_assignment(conn, execution_id, session_id, generation))
  691. return db._execute_write(write)
  692. _MESSAGE_FIELDS = frozenset({
  693. 'role', 'content', 'tool_name', 'tool_calls', 'tool_call_id', 'token_count',
  694. 'finish_reason', 'reasoning', 'reasoning_content', 'reasoning_details',
  695. 'codex_reasoning_items', 'codex_message_items', 'platform_message_id', 'message_id',
  696. 'observed', 'effect_disposition', '_compressed_summary', 'timestamp', 'api_content',
  697. 'display_kind', 'display_metadata', '_row_id', '_canonical_content', DB_ROW_SNAPSHOT, CANONICAL_ROW, DATA_USE,
  698. # Main's durable message identity (message_uid / merge witness / tool-call uids) and the alternation
  699. # repair's row counts ride every persisted dict; the owner's insert and coverage read them.
  700. }) | PERSISTENCE_ONLY_MESSAGE_FIELDS | {ABSORBED_ROW_IDS}
  701. # Compaction handoffs (archive/publish) also carry the summary's user-turn flag. This is the closed
  702. # boundary the owner validates AND the projection the worker facade applies to live dicts first.
  703. HANDOFF_MESSAGE_FIELDS = _MESSAGE_FIELDS | {'_compressed_summary_has_user_turn'}
  704. # Row state the owner's insert and transcript repair stamp on each message (main mutates the caller's
  705. # dict in place; a worker gets it back as an annotation). None = absent, so a stale adoption is cleared.
  706. # The owner-minted identity rides too: a worker dict without it re-mints a new uid on every later
  707. # archive/rotation copy, so the logical message changes identity across generations.
  708. _ROW_ANNOTATION_KEYS = ('_row_id', 'timestamp', DB_ROW_SNAPSHOT, CANONICAL_ROW,
  709. MESSAGE_UID, TOOL_CALL_UIDS, TOOL_CALL_UID)
  710. def row_annotations(messages):
  711. return [{key: msg.get(key) for key in _ROW_ANNOTATION_KEYS} for msg in messages]
  712. def _worker_append(db, conn, session_id, payload):
  713. if set(payload) - {'messages', 'turn_lease_holder'} or 'messages' not in payload:
  714. raise RuntimeStoreError('invalid_params')
  715. messages = payload['messages']
  716. if not isinstance(messages, list) or len(messages) > 1000:
  717. raise RuntimeStoreError('invalid_params')
  718. for msg in messages:
  719. if (not isinstance(msg, dict) or set(msg) - _MESSAGE_FIELDS
  720. or msg.get('role') not in ('user', 'assistant', 'system', 'tool')):
  721. raise RuntimeStoreError('invalid_params')
  722. # Existing row annotations can adopt only rows in THIS transcript.
  723. if '_row_id' in msg and not conn.execute(
  724. 'SELECT 1 FROM messages WHERE id=? AND session_id=?',
  725. (msg['_row_id'], session_id)).fetchone():
  726. raise RuntimeStoreError('permission_denied')
  727. holder = payload.get('turn_lease_holder')
  728. if holder is not None:
  729. _text(holder)
  730. count = db._append_messages_in_transaction(conn, session_id, messages, turn_lease_holder=holder)
  731. return {'count': count, 'annotations': row_annotations(messages)}
  732. def _worker_turn(db, conn, session_id, payload, operation):
  733. import time
  734. from hermes_state_compression import _claim_lease_row
  735. from hermes_state import _compression_lock_holder_process_is_dead
  736. allowed = {'holder'} if operation == 'turn.release' else {'holder', 'ttl_seconds'}
  737. if set(payload) != allowed:
  738. raise RuntimeStoreError('invalid_params')
  739. holder = _text(payload['holder'])
  740. ttl = payload.get('ttl_seconds', 300)
  741. if type(ttl) not in (float, int) or not 0.1 <= ttl <= 3600:
  742. raise RuntimeStoreError('invalid_params')
  743. key = db._session_turn_lease_key_on_conn(conn, session_id)
  744. now = time.time()
  745. if operation == 'turn.acquire':
  746. value = _claim_lease_row(conn, 'session_turn_leases', 'conversation_id', key,
  747. holder, now, now + ttl,
  748. lambda h, e: float(e) <= now or _compression_lock_holder_process_is_dead(h))[0]
  749. elif operation == 'turn.renew':
  750. value = conn.execute('UPDATE session_turn_leases SET expires_at=? WHERE conversation_id=? AND holder=?',
  751. (now + ttl, key, holder)).rowcount > 0
  752. else:
  753. conn.execute('DELETE FROM session_turn_leases WHERE conversation_id=? AND holder=?', (key, holder))
  754. value = None
  755. return {'value': value}
  756. def _worker_usage(db, conn, session_id, payload, *, auxiliary=False):
  757. from hermes_state_usage import _MODEL_USAGE_FIELDS, _TOKEN_COUNTERS
  758. source = None
  759. if not auxiliary:
  760. # ``source`` is the worker platform (turn_usage._agent_session_source). A Desktop
  761. # managed worker passes ``gui``; created_surface maps that to ``desktop``. The row
  762. # is usually stamped at admission. A compression child can still be NULL (or an
  763. # auxiliary-usage ``unknown``), and this worker never create_session's it, so fill
  764. # only an unset column, as the creator upsert does.
  765. raw = payload.get('source')
  766. source = raw if isinstance(raw, str) else None
  767. payload = {k: v for k, v in payload.items() if k != 'source'}
  768. allowed = (_MODEL_USAGE_FIELDS - {'billing_mode', 'actual_cost_usd', 'cost_status', 'cost_source'} | {'task'}) if auxiliary else (_MODEL_USAGE_FIELDS | {'pricing_version', 'absolute', 'task'})
  769. if set(payload) - allowed:
  770. raise RuntimeStoreError('invalid_params')
  771. for key, value in payload.items():
  772. if key in (*_TOKEN_COUNTERS, 'api_call_count'):
  773. valid = type(value) is int and 0 <= value <= 2**53
  774. elif key in ('estimated_cost_usd', 'actual_cost_usd'):
  775. valid = value is None or (type(value) in (int, float) and 0 <= value <= 1e12)
  776. elif key == 'absolute':
  777. valid = type(value) is bool
  778. else:
  779. valid = value is None or (isinstance(value, str) and len(value) <= 4096)
  780. if not valid:
  781. raise RuntimeStoreError('invalid_params')
  782. if auxiliary:
  783. if not payload.get('task'):
  784. raise RuntimeStoreError('invalid_params')
  785. db._record_model_usage(conn, session_id, **({'api_call_count': 1} | payload))
  786. else:
  787. db._update_token_counts_in_transaction(conn, session_id, **payload)
  788. # The managed worker's platform is the policy label (gui), not the in-process
  789. # agent surface (desktop). Stamp the same created_surface the insert uses.
  790. from gateway.session_policy import created_surface
  791. from hermes_state_compression import _UNSET_CREATED_SQL, stamp_compression_ancestors
  792. surfaced = created_surface(source)
  793. if surfaced:
  794. conn.execute(
  795. "UPDATE sessions SET created_source=? WHERE id=? AND "
  796. + _UNSET_CREATED_SQL.format(col="created_source"),
  797. (surfaced, session_id))
  798. stamp_compression_ancestors(conn, session_id)
  799. return {'value': None}
  800. def _worker_finish(db, conn, session_id, payload):
  801. if payload:
  802. raise RuntimeStoreError('invalid_params')
  803. return {'status': 'terminal'}
  804. def mutate_worker_execution(db, *, epoch, execution_id, session_id, generation,
  805. sequence, operation, payload):
  806. """One closed durable mutation and receipt; never call a self-committing API here."""
  807. from hermes_state_worker_context import (worker_context, worker_prompt, worker_row_role, worker_sidecars,
  808. worker_tail_role, worker_tool_names)
  809. from hermes_state_worker_compression import WORKER_COMPRESSION_HANDLERS
  810. from hermes_state_worker_lifecycle import WORKER_LIFECYCLE_HANDLERS
  811. handlers = {
  812. **WORKER_LIFECYCLE_HANDLERS,
  813. **WORKER_COMPRESSION_HANDLERS,
  814. 'session.context': worker_context,
  815. 'session.prompt': worker_prompt,
  816. 'session.sidecars': worker_sidecars,
  817. 'session.tools': worker_tool_names,
  818. 'session.tail_role': worker_tail_role,
  819. 'session.row_role': worker_row_role,
  820. 'transcript.append': _worker_append,
  821. 'execution.finish': _worker_finish,
  822. 'usage.main': _worker_usage,
  823. 'usage.auxiliary': lambda db, conn, sid, p: _worker_usage(db, conn, sid, p, auxiliary=True),
  824. **{name: (lambda db, conn, sid, p, op=name: _worker_turn(db, conn, sid, p, op))
  825. for name in ('turn.acquire', 'turn.renew', 'turn.release')},
  826. }
  827. return apply_worker_receipt(db, epoch=epoch, execution_id=execution_id, session_id=session_id,
  828. generation=generation, sequence=sequence, operation=operation,
  829. payload=payload, handlers=handlers)
  830. def apply_worker_receipt(db, *, epoch, execution_id, session_id, generation, sequence,
  831. operation, payload, handlers):
  832. """The one worker receipt transaction every ``worker.persist`` family shares.
  833. An exact (execution, sequence, digest) retry returns the stored result; otherwise the
  834. epoch, assignment, live status and next sequence are fenced, ``handlers[operation]`` runs on
  835. the transaction connection (it must not commit), and its result is stored as the receipt.
  836. ``execution.finish`` is the only operation that makes the execution terminal.
  837. """
  838. from hermes_state_worker_compression import worker_receipt_assignment
  839. if type(sequence) is not int or sequence < 1 or not isinstance(operation, str) or operation not in handlers:
  840. raise RuntimeStoreError('invalid_params')
  841. encoded = _json(payload)
  842. if len(encoded.encode('utf-8', errors='surrogatepass')) > 4 * 1024 * 1024:
  843. raise RuntimeStoreError('invalid_params')
  844. digest = admission_fingerprint(canonical_target=session_id,
  845. payload={'operation': operation, 'payload': json.loads(encoded)})
  846. def write(conn):
  847. _epoch(conn, epoch)
  848. row = worker_receipt_assignment(conn, execution_id, session_id, generation, sequence, digest)
  849. if row['owner_epoch'] != epoch:
  850. raise RuntimeStoreError('stale_epoch')
  851. old = conn.execute('SELECT * FROM worker_receipts WHERE execution_id=? AND sequence=?',
  852. (execution_id, sequence)).fetchone()
  853. if old is not None:
  854. if old['payload_digest'] != digest:
  855. raise RuntimeStoreError('admission_conflict')
  856. from hermes_state_runtime_workers import retired_receipt
  857. if retired_receipt(old['result_json']):
  858. raise RuntimeStoreError('stale_generation') # a terminal execution's read copy
  859. return json.loads(old['result_json'])
  860. if row['status'] not in ('registered', 'running'):
  861. raise RuntimeStoreError('stale_generation')
  862. if sequence != row['last_sequence'] + 1:
  863. raise RuntimeStoreError('invalid_params')
  864. # Each SQLite retry gets fresh rows; rolled-back annotations must not escape.
  865. changes = conn.total_changes
  866. result = handlers[operation](db, conn, session_id, json.loads(encoded))
  867. # The CAS fence moves only when the handler changed durable state (or the execution
  868. # closes): a read-only receipt (context, history, lineage, lifecycle) must not make a
  869. # client's mutation built on the pre-read revision fail revision_conflict.
  870. changed = conn.total_changes != changes or operation == 'execution.finish'
  871. conn.execute('INSERT INTO worker_receipts(execution_id,sequence,payload_digest,result_json) VALUES(?,?,?,?)',
  872. (execution_id, sequence, digest, json.dumps(result, ensure_ascii=True, allow_nan=False)))
  873. conn.execute("UPDATE worker_executions SET last_sequence=?,status=? WHERE execution_id=?",
  874. (sequence, 'terminal' if operation == 'execution.finish' else 'running', execution_id))
  875. if changed:
  876. conn.execute('UPDATE sessions SET runtime_revision=runtime_revision+1 WHERE id=?', (session_id,))
  877. if operation == 'execution.finish':
  878. from hermes_state_runtime_workers import compact_terminal_receipts
  879. compact_terminal_receipts(conn, execution_id)
  880. return result
  881. return db._execute_write(write, patience_s=db._TRANSCRIPT_WRITE_PATIENCE_S)