api_server_authority_runs.py 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189
  1. """API run controls resolve durable claims, never adapter agent/task ownership."""
  2. from dataclasses import asdict
  3. from gateway.session_contract import SessionActor, SessionRef
  4. from gateway.session_results import admission_result
  5. from hermes_state_runtime import RuntimeStoreError, _row
  6. def _authority(adapter):
  7. """The routed profile's authority: ``/p/<profile>/`` middleware already entered its scope."""
  8. from gateway.session_authorities import active_authority
  9. return active_authority(adapter.gateway_runner)
  10. # Malformed input is 400 and a retryable refusal 503; every other reason (ownership, generation,
  11. # identity) keeps the documented 409, as /v1/runs clients already branch on it. ``chain_moved``: a
  12. # chain's lookup went stale before its bind (``bind_api_steps``); a retry looks it up again.
  13. _STORE_ERROR_STATUS = {'invalid_params': 400, 'runtime_draining': 503, 'storage_unavailable': 503,
  14. 'chain_moved': 503}
  15. def store_error_response(_openai_error, exc):
  16. """A refused store call -> HTTP: a ``RuntimeStoreError`` keeps its reason; any other failure
  17. (SQLite writer patience exhausted, I/O) is ``storage_unavailable``."""
  18. from aiohttp import web
  19. reason = exc.reason if isinstance(exc, RuntimeStoreError) else 'storage_unavailable'
  20. if reason == 'admission_queued':
  21. # Accepted and still queued for the next owner, so not retryable: no Retry-After, and
  22. # x-should-retry tells the OpenAI/Anthropic SDKs not to retry the 503 on their own.
  23. return web.json_response(_openai_error(str(exc), err_type='server_error', code=reason), status=503,
  24. headers={'x-should-retry': 'false'})
  25. status = _STORE_ERROR_STATUS.get(reason, 409)
  26. return web.json_response(_openai_error(reason, code=reason), status=status,
  27. headers={'Retry-After': '1'} if status == 503 else None)
  28. async def continued_session(adapter, session_id, request_id=None):
  29. """``continued_api_session`` for an HTTP door, before its response starts -> ``(session_id,
  30. None)`` or ``(session_id, error response)``: a busy or unreadable store is the retryable 503
  31. ``storage_unavailable`` (``store_error_response``), as for the header-less chat lookup."""
  32. import asyncio
  33. import sqlite3
  34. from gateway.platforms.api_server import _openai_error
  35. from gateway.session_api import continued_api_session
  36. try:
  37. return await asyncio.to_thread(
  38. continued_api_session, adapter.gateway_runner, session_id, request_id), None
  39. except (RuntimeStoreError, sqlite3.Error, OSError) as exc:
  40. return session_id, store_error_response(_openai_error, exc)
  41. def run_admission(adapter, run_id):
  42. authority = _authority(adapter)
  43. if authority is None:
  44. return None
  45. # A streaming completion's public run id names its admission through the in-memory binding
  46. # its own request holds; every other run id is the admission's durable request id.
  47. alias = getattr(adapter, '_run_admission_aliases', {}).get(run_id)
  48. column, value = ('admission_id', alias) if alias else ('request_id', run_id)
  49. with authority.db._read_ctx() as conn:
  50. rows = conn.execute(f"SELECT * FROM session_admissions WHERE principal_id='api' AND {column}=?",
  51. (value,)).fetchall()
  52. if len(rows) > 1:
  53. raise RuntimeStoreError('admission_conflict')
  54. return (authority, _row(rows[0])) if rows else None
  55. def run_projection(adapter, run_id):
  56. owned = run_admission(adapter, run_id)
  57. if owned is None:
  58. return None
  59. authority, row = owned
  60. # ``unknown`` is not terminal: later turns wait for POST /v1/runs/{id}/resolve-unknown, and
  61. # ``interrupted`` is in TERMINAL_STATUSES, so a poller would stop at a paused head.
  62. status = {'queued': 'queued', 'started': 'running', 'unknown': 'unknown', 'terminal': row['outcome']}.get(row['status'])
  63. saved = admission_result(authority.db, row['admission_id'])
  64. result = saved.get('result', {}) if saved else {}
  65. if row['status'] == 'terminal':
  66. if result.get('interrupted') or row['outcome'] == 'interrupted':
  67. status = 'cancelled'
  68. elif result.get('failed') or result.get('error'):
  69. status = 'failed'
  70. elif row['outcome'] == 'completed' and saved:
  71. # The run SSE's own mapper: a `partial` or `completed: false` turn (iteration
  72. # budget, truncation) settles with outcome `completed` but is a failed run there,
  73. # so polling must not report `completed` for the same receipt.
  74. from gateway.platforms.api_server_runs import terminal_run_status
  75. status = terminal_run_status(result)[0]
  76. pending = []
  77. live = authority.sessions.get(row['target_session_id'])
  78. if live is not None and row['status'] == 'started':
  79. pending = list(live.controls.snapshot(row['target_session_id'], row['generation']))
  80. if any(prompt.get('kind') == 'approval' for prompt in pending):
  81. status = 'waiting_for_approval' # the documented run state main's run store reports
  82. return {'pending_controls': pending, 'run_id': run_id, 'status': status, 'session_id': row['target_session_id'],
  83. 'admission_id': row['admission_id'], 'execution_generation': row['generation'],
  84. 'output': result.get('final_response', ''), 'usage': saved.get('usage', {}) if saved else {}}
  85. async def send_clarify(adapter, *, chat_id, **kwargs):
  86. from gateway.platforms.base import SendResult
  87. authority = _authority(adapter)
  88. if authority is None or chat_id not in authority.sessions:
  89. return SendResult(success=False, error='No canonical API session')
  90. handle = authority._handle(SessionRef(authority.profile_id, chat_id))
  91. if handle.execution_state != 'running':
  92. return SendResult(success=False, error='No active API execution')
  93. # TurnRunner registers the shared prompt after this ACK; HTTP polling and WS
  94. # subscribers consume that projection rather than an adapter-local message.
  95. return SendResult(success=True, message_id=kwargs['clarify_id'])
  96. async def respond_run(adapter, run_id, body, *, kind):
  97. import uuid
  98. owned = run_admission(adapter, run_id)
  99. if owned is None:
  100. raise RuntimeStoreError('not_found')
  101. authority, row = owned
  102. generation = body.get('execution_generation')
  103. prompt_id = body.get('request_id')
  104. field = 'choice' if kind == 'approval' else 'answer'
  105. if set(body) != {'request_id', 'execution_generation', field}:
  106. raise RuntimeStoreError('invalid_params')
  107. if row['status'] != 'started' or type(generation) is not int or generation != row['generation']:
  108. raise RuntimeStoreError('stale_generation')
  109. ref = SessionRef(authority.profile_id, row['target_session_id'])
  110. actor = SessionActor('api', authority.profile_id,
  111. frozenset({'session:read', 'session:approve', 'session:respond'}), 'api-control:' + uuid.uuid4().hex)
  112. snapshot = await authority.attach(actor, ref, history=False) # prompts only
  113. try:
  114. if not any(p['prompt_id'] == prompt_id and p['kind'] == kind for p in snapshot.prompts):
  115. raise RuntimeStoreError('approval_not_pending')
  116. return await authority.respond(actor, ref, generation, prompt_id, {field: body[field]}, kind=kind)
  117. finally:
  118. await authority.detach(actor, snapshot.subscription_id)
  119. async def stop_run(adapter, run_id):
  120. owned = run_admission(adapter, run_id)
  121. if owned is None:
  122. raise RuntimeStoreError('not_found')
  123. authority, row = owned
  124. ref = SessionRef(authority.profile_id, row['target_session_id'])
  125. actor = SessionActor('api', authority.profile_id,
  126. frozenset({'session:submit', 'session:control'}), 'api-run:' + run_id)
  127. if row['status'] == 'queued':
  128. try:
  129. await authority.cancel_queued(actor, ref, row['admission_id'])
  130. except RuntimeStoreError as exc:
  131. # The drain claimed the head between the read above and the cancel commit: the run is
  132. # executing now, so Stop that exact claim below instead of refusing a live run.
  133. if exc.reason != 'stale_generation':
  134. raise
  135. authority, row = run_admission(adapter, run_id)
  136. else:
  137. adapter._stopping_run_ids.add(run_id)
  138. waiter = authority.waiters.pop(row['admission_id'], None)
  139. if waiter is not None and not waiter.done():
  140. waiter.set_result(None)
  141. return run_projection(adapter, run_id)
  142. if row['status'] == 'started':
  143. await authority.interrupt(actor, ref, row['generation'])
  144. adapter._stopping_run_ids.add(run_id)
  145. return {'run_id': run_id, 'status': 'stopping', 'admission_id': row['admission_id']}
  146. elif row['status'] == 'unknown':
  147. raise RuntimeStoreError('unknown_execution')
  148. return run_projection(adapter, run_id)
  149. async def resolve_unknown_run(adapter, run_id, body):
  150. """Resolve only the exact unknown admission durably bound to an owned API run."""
  151. owned = run_admission(adapter, run_id)
  152. if owned is None:
  153. raise RuntimeStoreError('not_found')
  154. authority, row = owned
  155. if not isinstance(body, dict) or set(body) != {'admission_id', 'execution_generation'}:
  156. raise RuntimeStoreError('invalid_params')
  157. if body['admission_id'] != row['admission_id']:
  158. raise RuntimeStoreError('not_found')
  159. generation = body['execution_generation']
  160. # Status/generation are the store's decision (``resolve_unknown_session_input``): it also replays the
  161. # committed receipt to an exact retry whose acknowledgement was lost, which a pre-check here refused.
  162. actor = SessionActor(
  163. 'api', authority.profile_id, frozenset({'session:submit', 'session:control'}),
  164. 'api-run:' + run_id)
  165. receipt = await authority.resolve_unknown(
  166. actor, SessionRef(authority.profile_id, row['target_session_id']),
  167. row['admission_id'], generation)
  168. return asdict(receipt)