session_hosted_rpc.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343
  1. """Private hosted-room producer over the gateway's canonical local-session FIFO.
  2. One instance belongs to a durable room/member/profile binding. The service supplies
  3. its principal and current-membership check; neither is taken from RPC payloads.
  4. """
  5. from __future__ import annotations
  6. import asyncio
  7. from dataclasses import asdict, replace
  8. from functools import partial
  9. import hashlib
  10. import json
  11. from gateway.hosted_room_driver import TaskIdentity
  12. from gateway.owner_identity import same_owner
  13. from gateway.session_contract import SessionRef, Submission
  14. from hermes_state_runtime import RuntimeStoreError, list_session_admissions
  15. _RESULTLESS_OUTCOMES = frozenset({'interrupted', 'cancelled'})
  16. # Encoded receipts per ``results`` read; under the 512 KiB private-socket response line.
  17. _RESULTS_BUDGET_BYTES = 384 * 1024
  18. class HostedRoomAuthorityRPC:
  19. accepts_attachments = True # submit resolves event-bound bytes per member
  20. def __init__(self, authority, loop, *, room_id, member_id, profile, principal,
  21. authorize, authorize_write=None, timeout=30):
  22. self.authority, self.loop = authority, loop
  23. self.room_id, self.member_id, self.profile = room_id, member_id, profile
  24. self.principal, self.authorizer, self.timeout = principal, authorize, timeout
  25. self.authorize_write = authorize_write
  26. self.callbacks = {}
  27. binding = json.dumps([room_id, member_id, profile], separators=(',', ':'))
  28. self.creation_id = 'hosted:' + hashlib.sha256(binding.encode()).hexdigest()
  29. from gateway.session_local_recovery import local_identity
  30. self.ref = SessionRef(authority.profile_id, local_identity(
  31. authority.profile_id, principal.subject, self.creation_id))
  32. def _call(self, operation, **params):
  33. try:
  34. running = asyncio.get_running_loop()
  35. except RuntimeError:
  36. running = None
  37. if running is self.loop:
  38. raise RuntimeStoreError('invalid_params')
  39. if not self.loop.is_running():
  40. raise RuntimeStoreError('runtime_draining')
  41. future = asyncio.run_coroutine_threadsafe(self._dispatch(operation, params), self.loop)
  42. # A timeout is ambiguous: never cancel an admission or claim not_admitted.
  43. return future.result(self.timeout)
  44. async def _dispatch(self, operation, params):
  45. from gateway.session_authorities import owner_scope
  46. with owner_scope(self.authority):
  47. return await self._dispatch_owned(operation, params)
  48. async def _dispatch_owned(self, operation, params):
  49. if (params.get('profile', self.profile) != self.profile
  50. or self.principal.profile_id != self.authority.profile_id):
  51. raise RuntimeStoreError('profile_mismatch')
  52. if (params.get('source', 'bot_room') != 'bot_room'
  53. or params.get('session_id', self.ref.session_id) != self.ref.session_id
  54. or params.get('title', f'Group: {self.room_id}') != f'Group: {self.room_id}'):
  55. raise RuntimeStoreError('permission_denied')
  56. task, generation = params.get('task'), params.get('execution_generation')
  57. if operation == 'submit' and (not isinstance(task, TaskIdentity)
  58. or task.room_id != self.room_id or type(generation) is not int or generation < 1):
  59. raise RuntimeStoreError('invalid_params')
  60. if self.authorizer(operation, task, generation) is not True:
  61. raise RuntimeStoreError('permission_denied')
  62. return await getattr(self, '_' + operation)(params)
  63. async def _resolve_exact(self, params):
  64. if self.authority.db.get_session(self.ref.session_id) is None:
  65. return None
  66. return await self._resume(params)
  67. async def _create(self, params):
  68. if self.authority.db.get_session(self.ref.session_id) is not None:
  69. return await self._resume(params)
  70. from gateway.run import _load_gateway_config, _resolve_gateway_model
  71. from gateway.session_policy import build_policy
  72. from gateway.session_local import stage_local_session
  73. from gateway.session_local_recovery import restore_local_session
  74. from gateway.session_runtime_workers import tracked_write
  75. from hermes_state_local import commit_local_session
  76. from agent.skill_utils import parse_config_string_list
  77. private = {}
  78. config = _load_gateway_config()
  79. # A room has no clarification reply (group controls expose approvals only), so a member
  80. # turn that called clarify would block for clarify_timeout with its question unseen. As
  81. # for cron, freeze it out; the member asks its question in the room reply instead.
  82. agent = dict(config.get('agent') or {})
  83. agent['disabled_toolsets'] = sorted(
  84. set(parse_config_string_list(agent.get('disabled_toolsets'))) | {'clarify'})
  85. policy = build_policy({'source': 'gui'}, {**config, 'agent': agent}, private_secrets=private)
  86. policy = replace(policy, source='bot_room', platform='bot_room',
  87. model=policy.model or _resolve_gateway_model(policy.config()),
  88. toolsets=tuple(sorted(set(policy.toolsets) | {'bot_room'})))
  89. staged = stage_local_session(self.authority, self.principal, {'request_id': self.creation_id},
  90. trusted_policy=policy, trusted_secrets=private)
  91. # Creation commit and label run off the owner loop (N90); a concurrent duplicate converges.
  92. if isinstance(staged, dict):
  93. await tracked_write(self.authority, partial(
  94. commit_local_session, self.authority.db, epoch=self.authority.epoch, receipt=staged))
  95. restore_local_session(self.authority, staged['session_id'])
  96. from hermes_state_runtime import _epoch
  97. def label(conn):
  98. _epoch(conn, self.authority.epoch)
  99. conn.execute('UPDATE sessions SET title=?, hidden=1 WHERE id=?',
  100. (f'Group: {self.room_id}', self.ref.session_id))
  101. await tracked_write(self.authority, partial(self.authority.db._execute_write, label))
  102. return await self._resume(params)
  103. async def _resume(self, params):
  104. await self.authority.resolve(self.principal, self.ref)
  105. return {'session_id': self.ref.session_id, 'title': f'Group: {self.room_id}'}
  106. def _rows(self):
  107. self.authority.authorize(self.principal, self.ref, 'session:read')
  108. rows = list_session_admissions(self.authority.db, session_id=self.ref.session_id, pending_only=False)
  109. result = []
  110. for row in rows:
  111. if not same_owner(row['principal_id'], self.principal.subject) or not row['request_id'].startswith('hosted:'):
  112. continue
  113. try:
  114. task, generation = json.loads(row['request_id'][7:])
  115. identity = TaskIdentity(**task)
  116. if identity.room_id != self.room_id or type(generation) is not int:
  117. raise ValueError('invalid hosted identity')
  118. except (TypeError, ValueError, KeyError) as exc:
  119. raise RuntimeStoreError('storage_unavailable') from exc
  120. result.append((row, identity, generation))
  121. return result
  122. def _terminal(self, row, task, generation):
  123. from gateway.session_results import admission_result
  124. saved = admission_result(self.authority.db, row['admission_id'])
  125. # Unknown discard and queued cancellation never ran: no result exists to recover.
  126. if saved is None and row['outcome'] not in _RESULTLESS_OUTCOMES:
  127. raise RuntimeStoreError('storage_unavailable')
  128. value = saved['result'] if saved is not None else {}
  129. status = {'completed': 'settled', 'interrupted': 'cancelled', 'cancelled': 'cancelled'}.get(row['outcome'], 'failed')
  130. receipt = {'status': status, 'text': value.get('final_response', ''),
  131. 'message_id': row['admission_id'], 'settlement_id': row['admission_id'],
  132. 'task_id': task.task_id, 'execution_generation': generation}
  133. callback = self.callbacks.pop(row['admission_id'], None)
  134. if callback is not None:
  135. callback(receipt)
  136. return receipt
  137. async def _submit(self, params):
  138. task, generation = params['task'], params['execution_generation']
  139. request_id = 'hosted:' + json.dumps([asdict(task), generation], sort_keys=True, separators=(',', ':'))
  140. # Refuse unknown before submit: submit itself schedules the queue on retries.
  141. rows = self._rows()
  142. if any(row['status'] == 'unknown' for row, _, _ in rows):
  143. raise RuntimeStoreError('unknown_execution')
  144. from gateway.session_hosted_attachments import prepare_before_submit, submission_payload
  145. from gateway.session_ingress_media import capture_lease
  146. # One lease from capture through the admission commit: documents ride in the prompt text,
  147. # so submit's own lease does not hold them, and an older row's release could unlink a
  148. # shared content-addressed capture in between.
  149. with capture_lease():
  150. payload = await asyncio.to_thread(prepare_before_submit, partial(
  151. submission_payload, self, params['prompt'], params.get('attachments')))
  152. authorize_write = self.authorize_write
  153. receipt = await self.authority.submit(self.principal, Submission(
  154. request_id, self.ref, payload, 'queue'),
  155. _authorize_write=(lambda conn: authorize_write(conn, task, generation))
  156. if authorize_write is not None else None)
  157. self.callbacks[receipt.admission_id] = params['on_terminal']
  158. if receipt.status in {'queued', 'started'}:
  159. waiter = self.authority.waiters.get(receipt.admission_id)
  160. if waiter is None:
  161. waiter = self.loop.create_future()
  162. self.authority.waiters[receipt.admission_id] = waiter
  163. waiter.add_done_callback(lambda done: self._completed(receipt.admission_id, done))
  164. if receipt.status == 'terminal':
  165. for row, identity, gen in self._rows():
  166. if row['admission_id'] == receipt.admission_id:
  167. self._terminal(row, identity, gen)
  168. return asdict(receipt)
  169. def _completed(self, admission_id, future):
  170. if future.cancelled() or admission_id not in self.callbacks:
  171. return
  172. if future.exception() is not None:
  173. # A paused FIFO remains queued/unknown; polling or an explicit
  174. # resolution will publish its eventual durable terminal outcome.
  175. return
  176. if self.authorizer('terminal', None, None) is not True:
  177. self.callbacks.pop(admission_id, None)
  178. return
  179. for row, task, generation in self._rows():
  180. if row['admission_id'] == admission_id and row['status'] == 'terminal':
  181. self._terminal(row, task, generation)
  182. async def _history(self, params):
  183. from gateway.session_local_recovery import local_history, prepare_history
  184. # Authorize (restoring a cold room session) before the read names its live entry, and again after
  185. # the await (``_rows``): a delete in between answers not_found, never a KeyError.
  186. self.authority.authorize(self.principal, self.ref, 'session:read')
  187. prepared = await prepare_history(self.authority, self.ref)
  188. rows = self._rows()
  189. history = self.authority.unless_deleted(
  190. self.ref, lambda: list(local_history(self.authority, self.ref, prepared)))
  191. for row, task, generation in rows:
  192. if row['status'] == 'terminal':
  193. receipt = self._terminal(row, task, generation)
  194. history.append({**receipt, 'role': 'assistant', 'content': receipt['text']})
  195. return history
  196. async def _results(self, params):
  197. """Terminal receipts only, never the transcript: one bounded response line.
  198. ``admission_ids`` narrows the read to the caller's pending callbacks. Each text
  199. keeps just over the driver's persisted bound, so its own truncation (and notice)
  200. is unchanged; newest receipts win the byte budget and the rest follow next poll.
  201. """
  202. wanted = params.get('admission_ids')
  203. if wanted is not None and (not isinstance(wanted, list) or len(wanted) > 256
  204. or not all(isinstance(i, str) and i for i in wanted)):
  205. raise RuntimeStoreError('invalid_params')
  206. from tui_gateway.hosted_room_driver import MAX_TERMINAL_TEXT_BYTES
  207. receipts, used = [], 0
  208. for row, task, generation in reversed(self._rows()):
  209. if row['status'] != 'terminal' or (wanted is not None and row['admission_id'] not in wanted):
  210. continue
  211. receipt = self._terminal(row, task, generation)
  212. raw = receipt['text'].encode('utf-8')
  213. if len(raw) > MAX_TERMINAL_TEXT_BYTES:
  214. receipt['text'] = raw[:MAX_TERMINAL_TEXT_BYTES + 4].decode('utf-8', 'ignore')
  215. used += len(json.dumps(receipt))
  216. if receipts and used > _RESULTS_BUDGET_BYTES:
  217. break
  218. receipts.append(receipt)
  219. return receipts[::-1]
  220. async def _info(self, params):
  221. rows = self._rows()
  222. for row, task, generation in rows:
  223. if row['status'] == 'terminal':
  224. self._terminal(row, task, generation)
  225. current = next(((row, task, generation) for row, task, generation in rows if row['status'] in {'started', 'unknown', 'queued'}), None)
  226. result = {'active': current is not None and current[0]['status'] != 'unknown',
  227. 'task_id': current[1].task_id if current else None,
  228. 'execution_generation': current[2] if current else None,
  229. 'status': current[0]['status'] if current else 'idle'}
  230. live = self.authority.sessions.get(self.ref.session_id)
  231. if current is not None and current[0]['status'] == 'started' and live is not None:
  232. # The member's current tool and last frame, for the room's "scout · terminal · 7s".
  233. result.update(live.event_stream.progress_for(current[0]['generation']))
  234. snapshot = await self.authority.attach(self.principal, self.ref, history=False) # prompts only
  235. if snapshot.prompts:
  236. prompt = next((p for p in snapshot.prompts if p.get('kind') == 'approval'), None)
  237. if prompt:
  238. result.update(status='waiting_for_approval', pending_approval={
  239. **prompt, 'request_id': prompt['prompt_id'],
  240. 'choices': [c for c in prompt['choices'] if c in {'once', 'deny'}]})
  241. return result
  242. async def _interrupt(self, params):
  243. rows = self._rows()
  244. current = next(((row, task) for row, task, _ in rows if row['status'] in {'started', 'unknown', 'queued'}), None)
  245. if current is None or current[1].task_id != params['expected_task_id']:
  246. raise RuntimeStoreError('stale_generation')
  247. row, _ = current
  248. if row['status'] == 'unknown':
  249. raise RuntimeStoreError('unknown_execution')
  250. if row['status'] == 'queued':
  251. row = await self.authority.cancel_or_claimed(self.principal, self.ref, row['admission_id'])
  252. if row is not None:
  253. await self.authority.interrupt(self.principal, self.ref, row['generation'])
  254. return {'interrupted': True, 'status': 'interrupted'}
  255. async def _discard(self, params):
  256. generation = params['execution_generation']
  257. if type(generation) is not int or generation < 1:
  258. raise RuntimeStoreError('invalid_params')
  259. matches = [(row, task) for row, task, hosted_generation in self._rows()
  260. if (row['status'] == 'unknown' or (row['status'] == 'terminal' and row['outcome'] == 'interrupted'))
  261. and task.task_id == params['expected_task_id']
  262. and hosted_generation == generation]
  263. if len(matches) != 1:
  264. raise RuntimeStoreError('stale_generation')
  265. row, task = matches[0]
  266. # The public fence is hosted; the canonical CAS uses its own generation.
  267. if row['status'] == 'unknown':
  268. await self.authority.resolve_unknown(
  269. self.principal, self.ref, row['admission_id'], row['generation'])
  270. return {'discarded': True, 'status': 'cancelled', 'task_id': task.task_id,
  271. 'execution_generation': generation}
  272. async def _approve(self, params):
  273. if params['choice'] not in {'once', 'deny'}:
  274. raise RuntimeStoreError('invalid_params')
  275. snapshot = await self.authority.attach(self.principal, self.ref, history=False) # prompts only
  276. prompt = next((p for p in snapshot.prompts if p.get('prompt_id') == params['request_id']
  277. and p.get('kind') == 'approval'), None)
  278. if prompt is None:
  279. raise RuntimeStoreError('stale_generation')
  280. return await self.authority.respond(self.principal, self.ref,
  281. snapshot.handle.execution_generation, params['request_id'], {'choice': params['choice']})
  282. def resolve_exact(self, *, profile, title, source):
  283. return self._call('resolve_exact', profile=profile, title=title, source=source)
  284. def create(self, *, profile, title, source):
  285. return self._call('create', profile=profile, title=title, source=source)
  286. def resume(self, *, profile, session_id, source):
  287. return self._call('resume', profile=profile, session_id=session_id, source=source)
  288. def submit(self, *, profile, session_id, prompt, source, task, execution_generation, on_terminal,
  289. member_id=None, attachments=None):
  290. # One producer per member binding: a task for another member reached the wrong producer.
  291. if member_id is not None and member_id != self.member_id:
  292. raise RuntimeStoreError('permission_denied')
  293. return self._call('submit', profile=profile, session_id=session_id, source=source,
  294. prompt=prompt, task=task, execution_generation=execution_generation, on_terminal=on_terminal,
  295. attachments=attachments)
  296. def history(self, *, profile, session_id, source):
  297. return self._call('history', profile=profile, session_id=session_id, source=source)
  298. def results(self, *, profile, session_id, source, admission_ids=None):
  299. return self._call('results', profile=profile, session_id=session_id, source=source,
  300. admission_ids=admission_ids)
  301. def info(self, *, profile, session_id, source):
  302. return self._call('info', profile=profile, session_id=session_id, source=source)
  303. def interrupt(self, *, profile, session_id, source, expected_task_id):
  304. return self._call('interrupt', profile=profile, session_id=session_id, source=source, expected_task_id=expected_task_id)
  305. def discard(self, *, profile, session_id, source, expected_task_id, execution_generation):
  306. return self._call('discard', profile=profile, session_id=session_id, source=source,
  307. expected_task_id=expected_task_id, execution_generation=execution_generation)
  308. def approve(self, *, session_id, request_id, choice):
  309. return self._call('approve', session_id=session_id, request_id=request_id, choice=choice)