session_worker_delegation.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231
  1. """Closed delegation-ledger operations in the existing worker receipt transaction.
  2. The caller must supply the verified registration's execution/PID/birth, never
  3. operation payload identity. No handler opens a database or commits independently.
  4. """
  5. from functools import partial
  6. import json
  7. import math
  8. import time
  9. from hermes_state_runtime import RuntimeStoreError
  10. _TASK_KEYS = {'goal', 'goals', 'context', 'toolsets', 'role', 'model', 'is_batch', 'task_indexes'}
  11. _ROUTE = {'session_key': 'origin_session', 'parent_session_id': 'parent_session_id',
  12. 'origin_ui_session_id': 'origin_ui_session_id', 'origin_session_id': 'origin_session_id'}
  13. _EVENT_KEYS = _TASK_KEYS | set(_ROUTE) | {
  14. 'type', 'delegation_id', 'status', 'completed_at', 'dispatched_at', 'summary', 'error',
  15. 'api_calls', 'duration_seconds', 'total_duration_seconds', 'results', 'live_transcripts',
  16. 'group', 'exit_reason', 'stalled_after_quiet_seconds', 'stall_threshold_seconds',
  17. 'stall_phase', 'stall_grace_seconds'}
  18. def _shape(payload, keys):
  19. if not isinstance(payload, dict) or set(payload) != set(keys):
  20. raise RuntimeStoreError('invalid_params')
  21. def _text(value):
  22. if not isinstance(value, str) or not value or len(value) > 512:
  23. raise RuntimeStoreError('invalid_params')
  24. def _number(value):
  25. if type(value) not in (int, float) or not math.isfinite(value) or value < 0:
  26. raise RuntimeStoreError('invalid_params')
  27. def _owned(conn, session_id, execution_id, delegation_id):
  28. _text(delegation_id)
  29. row = conn.execute('SELECT * FROM async_delegations WHERE delegation_id=?', (delegation_id,)).fetchone()
  30. if row is None:
  31. raise RuntimeStoreError('not_found')
  32. if row['parent_session_id'] != session_id or row['owner_execution_id'] != execution_id:
  33. raise RuntimeStoreError('permission_denied')
  34. return row
  35. def _dispatch(db, conn, session_id, payload, *, execution_id, worker_pid, worker_birth):
  36. _shape(payload, {'delegation_id', 'task', 'dispatched_at', *_ROUTE})
  37. _text(payload['delegation_id'])
  38. _number(payload['dispatched_at'])
  39. if not isinstance(payload['task'], dict) or set(payload['task']) - _TASK_KEYS:
  40. raise RuntimeStoreError('invalid_params')
  41. session = conn.execute('SELECT source,session_key FROM sessions WHERE id=?', (session_id,)).fetchone()
  42. # Only compute registrations are currently enabled. Native/child origins need
  43. # an owner-reserved source envelope, not worker-selected transport identities.
  44. if session is None or session['source'] not in ('cli', 'tui', 'gui', 'tool', 'oneshot'):
  45. raise RuntimeStoreError('unsupported_producer')
  46. if (payload['parent_session_id'] != session_id
  47. or payload['session_key'] != (session['session_key'] or '')
  48. or payload['origin_ui_session_id'] not in ('', session_id)
  49. or payload['origin_session_id'] not in ('', session_id)):
  50. raise RuntimeStoreError('permission_denied')
  51. old = conn.execute('SELECT 1 FROM async_delegations WHERE delegation_id=?', (payload['delegation_id'],)).fetchone()
  52. if old is not None:
  53. _owned(conn, session_id, execution_id, payload['delegation_id'])
  54. # Exact network retries are resolved by worker_receipts before this handler.
  55. raise RuntimeStoreError('admission_conflict')
  56. import psutil
  57. from gateway.status import get_process_start_time
  58. try:
  59. from hermes_cli import process_identity
  60. if process_identity.worker_birth(psutil.Process(worker_pid)) != worker_birth:
  61. raise RuntimeStoreError('permission_denied')
  62. owner_started_at = get_process_start_time(worker_pid)
  63. except psutil.Error as exc:
  64. raise RuntimeStoreError('worker_not_live') from exc
  65. now = time.time()
  66. conn.execute('''INSERT INTO async_delegations
  67. (delegation_id,origin_session,origin_ui_session_id,parent_session_id,state,dispatched_at,
  68. updated_at,owner_pid,owner_started_at,task_json,origin_session_id,owner_execution_id)
  69. VALUES(?,?,?,?,'running',?,?,?,?,?,?,?)''',
  70. (payload['delegation_id'], payload['session_key'], payload['origin_ui_session_id'], session_id,
  71. payload['dispatched_at'], now, worker_pid, owner_started_at, json.dumps(payload['task']),
  72. payload['origin_session_id'], execution_id))
  73. return {'value': None}
  74. def _complete(db, conn, session_id, payload, *, execution_id):
  75. _shape(payload, {'event', 'result'})
  76. event = payload['event']
  77. if (not isinstance(event, dict) or set(event) - _EVENT_KEYS
  78. or not isinstance(payload['result'], dict) or event.get('type') != 'async_delegation'
  79. or event.get('status') not in ('completed', 'success', 'error', 'interrupted', 'stalled', 'unknown')):
  80. raise RuntimeStoreError('invalid_params')
  81. row = _owned(conn, session_id, execution_id, event.get('delegation_id'))
  82. if any(event.get(key, '') != (row[column] or '') for key, column in _ROUTE.items()):
  83. raise RuntimeStoreError('permission_denied')
  84. _number(event.get('completed_at'))
  85. if row['state'] != 'running':
  86. raise RuntimeStoreError('admission_conflict')
  87. conn.execute('''UPDATE async_delegations SET state=?,completed_at=?,updated_at=?,event_json=?,result_json=?
  88. WHERE delegation_id=?''', (event['status'], event['completed_at'], time.time(),
  89. json.dumps(event), json.dumps(payload['result']), event['delegation_id']))
  90. return {'value': None}
  91. def _child(db, conn, session_id, payload, *, execution_id):
  92. _shape(payload, {'delegation_id', 'entry'})
  93. row = _owned(conn, session_id, execution_id, payload['delegation_id'])
  94. entry = payload['entry']
  95. if not isinstance(entry, dict) or type(entry.get('task_index')) is not int or entry['task_index'] < 0:
  96. raise RuntimeStoreError('invalid_params')
  97. if row['state'] != 'running':
  98. return {'value': None}
  99. previous = json.loads(row['result_json'] or '{}')
  100. results = [item for item in previous.get('results', []) if item.get('task_index') != entry['task_index']]
  101. results.append(entry)
  102. conn.execute('UPDATE async_delegations SET result_json=?,updated_at=? WHERE delegation_id=?',
  103. (json.dumps({'results': results, 'partial': True}), time.time(), payload['delegation_id']))
  104. return {'value': None}
  105. def _read(db, conn, session_id, payload, *, execution_id):
  106. _shape(payload, {'delegation_id'})
  107. row = _owned(conn, session_id, execution_id, payload['delegation_id'])
  108. return {'value': {key: row[key] for key in ('delegation_id', 'origin_session', 'state', 'dispatched_at',
  109. 'completed_at', 'delivery_state', 'delivery_attempts', 'origin_session_id')} |
  110. {'result': json.loads(row['result_json']) if row['result_json'] else None}}
  111. def _claim(db, conn, session_id, payload, *, execution_id):
  112. _shape(payload, {'delegation_id', 'claim_id'})
  113. row = _owned(conn, session_id, execution_id, payload['delegation_id'])
  114. _text(payload['claim_id'])
  115. now = time.time()
  116. if row['state'] in ('running', 'finalizing'):
  117. return {'value': False}
  118. changed = conn.execute('''UPDATE async_delegations SET delivery_claim=?,delivery_claimed_at=?,
  119. delivery_attempts=delivery_attempts+1,updated_at=? WHERE delegation_id=? AND delivery_state='pending'
  120. AND (delivery_claim IS NULL OR delivery_claimed_at < ?)''',
  121. (payload['claim_id'], now, now, payload['delegation_id'], now - 300)).rowcount
  122. return {'value': changed == 1}
  123. _DELIVERY_UPDATES = {
  124. 'ack': "delivery_state='delivered',delivered_at=:now",
  125. 'drop': "delivery_state='dropped'",
  126. 'defer': 'delivery_attempts=MAX(0,delivery_attempts-1)',
  127. 'release': "delivery_state=CASE WHEN delivery_attempts>=:cap THEN 'dropped' ELSE delivery_state END",
  128. }
  129. def _delivery(db, conn, session_id, payload, *, execution_id, action):
  130. from tools.async_delegation import _MAX_DELIVERY_ATTEMPTS
  131. _shape(payload, {'delegation_id', 'claim_id'})
  132. _owned(conn, session_id, execution_id, payload['delegation_id'])
  133. _text(payload['claim_id'])
  134. # Only constant, closed statement fragments enter this query.
  135. changed = conn.execute('UPDATE async_delegations SET ' + _DELIVERY_UPDATES[action] +
  136. ",updated_at=:now,delivery_claim=NULL,delivery_claimed_at=NULL WHERE delegation_id=:delegation_id "
  137. "AND delivery_state='pending' AND delivery_claim=:claim_id",
  138. payload | {'now': time.time(), 'cap': _MAX_DELIVERY_ATTEMPTS}).rowcount
  139. return {'value': changed == 1}
  140. def _unscheduled(db, conn, session_id, payload, *, execution_id):
  141. _shape(payload, {'delegation_id'})
  142. row = _owned(conn, session_id, execution_id, payload['delegation_id'])
  143. if row['state'] != 'running' or row['result_json'] is not None:
  144. raise RuntimeStoreError('admission_conflict')
  145. conn.execute('DELETE FROM async_delegations WHERE delegation_id=?', (payload['delegation_id'],))
  146. return {'value': None}
  147. def worker_delegation_handlers(execution_id, worker_pid, worker_birth):
  148. """Register these handlers only after gateway worker identity verification."""
  149. _text(execution_id)
  150. if type(worker_pid) is not int or worker_pid <= 0:
  151. raise RuntimeStoreError('invalid_params')
  152. _number(worker_birth)
  153. handlers = {name: partial(fn, execution_id=execution_id) for name, fn in {
  154. 'complete': _complete, 'child': _child, 'read': _read, 'claim': _claim,
  155. 'unscheduled': _unscheduled}.items()}
  156. handlers['dispatch'] = partial(_dispatch, execution_id=execution_id, worker_pid=worker_pid, worker_birth=worker_birth)
  157. handlers.update({name: partial(_delivery, execution_id=execution_id, action=name) for name in _DELIVERY_UPDATES})
  158. return {'delegation.' + name: handler for name, handler in handlers.items()}
  159. def persist_worker_delegation(db, *, epoch, execution_id, session_id, generation,
  160. sequence, operation, payload, worker_pid, worker_birth):
  161. """Closed ledger sibling of mutate_worker_execution, sharing its receipt stream.
  162. Gateway calls this ONLY after its ordinary worker claim verification. The
  163. verified process identity is deliberately separate from the operation payload.
  164. """
  165. from hermes_state_runtime import apply_worker_receipt
  166. handlers = worker_delegation_handlers(execution_id, worker_pid, worker_birth)
  167. return apply_worker_receipt(db, epoch=epoch, execution_id=execution_id, session_id=session_id,
  168. generation=generation, sequence=sequence, operation=operation,
  169. payload=payload, handlers=handlers)
  170. async def worker_delegation_request(connection, ref, params):
  171. """worker.persist delegation-family hook; all authentication stays mandatory."""
  172. import asyncio
  173. from gateway.session_worker import _SCOPE, _claim, _verify
  174. if set(params) != _SCOPE | {'epoch', 'sequence', 'operation', 'payload'}:
  175. raise RuntimeStoreError('invalid_params')
  176. claim = _claim(connection, ref, params)
  177. authority = connection.authority
  178. if 'worker:adopt' not in connection.actor.capabilities:
  179. raise RuntimeStoreError('permission_denied')
  180. _verify(connection, ref, params, claim)
  181. def persist_and_publish():
  182. result = persist_worker_delegation(authority.db,
  183. **{key: params[key] for key in ('epoch', 'execution_id', 'session_id', 'generation',
  184. 'sequence', 'operation', 'payload')},
  185. worker_pid=params['pid'], worker_birth=params['birth'])
  186. if params['operation'] == 'delegation.complete':
  187. with authority.db._read_ctx() as conn:
  188. row = _owned(conn, ref.session_id, params['execution_id'], params['payload']['event']['delegation_id'])
  189. event = json.loads(row['event_json']) if row['delivery_state'] == 'pending' else None
  190. if event is not None:
  191. from tools.process_registry import process_registry
  192. # Queueing after commit can repeat after a lost ACK. Existing
  193. # token-guarded claims arbitrate delivery; never ACK at publication.
  194. process_registry.completion_queue.put(event | {'restored': True})
  195. return result
  196. return await asyncio.to_thread(persist_and_publish)