session_hosted_transport.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375
  1. """Hosted producers between independently owned local profile daemons.
  2. Only the private OS-authenticated owner socket exposes these verbs. The source
  3. owner attests durable membership/task state; the target never opens its database.
  4. """
  5. from __future__ import annotations
  6. import asyncio
  7. import base64
  8. from dataclasses import asdict
  9. import hashlib
  10. import json
  11. import os
  12. from pathlib import Path
  13. import threading
  14. import time
  15. from gateway.hosted_room_driver import TaskIdentity
  16. from gateway.session_contract import SessionActor
  17. from gateway.session_authorities import owner_scope
  18. from gateway.session_hosted_rpc import HostedRoomAuthorityRPC
  19. from gateway.owner_identity import same_owner
  20. from hermes_state_runtime import RuntimeStoreError, _epoch
  21. _BINDING = 'gateway.hosted.transport.v1:'
  22. _OPERATIONS = frozenset({'resolve_exact', 'create', 'resume', 'submit', 'history', 'results',
  23. 'info', 'interrupt', 'discard', 'approve'})
  24. # One chunk per private-socket exchange. The response is a single JSON line capped at
  25. # gateway.control_socket._MAX_RESPONSE_BYTES (512 KiB) on both the POSIX socket and the
  26. # Windows pipe: 360 KiB raw -> 480 KiB base64, leaving 32 KiB for the envelope (owner
  27. # subject, target home, digest); the attachment attest result carries no prompt/manifest.
  28. _CHUNK_BYTES = 360 * 1024
  29. _CAPS = frozenset({'session:create', 'session:read', 'session:submit',
  30. 'session:control', 'session:approve'})
  31. def owner_request(home, verb, params, *, timeout=30):
  32. """Authenticated private socket exchange; no owner-start or storage fallback."""
  33. home = Path(home)
  34. if home != home.resolve():
  35. raise RuntimeStoreError('permission_denied')
  36. from hermes_cli.gateway_runtime import discover_gateway_endpoint, control_home_for
  37. deadline = time.monotonic() + timeout
  38. discovered = discover_gateway_endpoint(home, timeout=timeout)
  39. if discovered.state != 'ready' or discovered.endpoint is None:
  40. raise RuntimeStoreError('runtime_draining')
  41. from hermes_constants import hermes_home_key
  42. if hermes_home_key(discovered.endpoint.profile_id) != hermes_home_key(home):
  43. raise RuntimeStoreError('profile_mismatch')
  44. # Socket identity and logical authority identity are distinct under multiplex.
  45. params = {**params, 'profile_id': discovered.endpoint.profile_id}
  46. home = control_home_for(home, discovered.endpoint)
  47. timeout = deadline - time.monotonic()
  48. if timeout <= 0:
  49. raise RuntimeStoreError('runtime_draining')
  50. request = json.dumps({'protocol': 1, 'id': 1, 'verb': verb, 'params': params}).encode() + b'\n'
  51. if len(request) > 65536:
  52. raise RuntimeStoreError('request_too_large')
  53. if os.name == 'nt':
  54. from gateway.runtime_bootstrap_windows import query_runtime_control
  55. raw = query_runtime_control(home, request, timeout)
  56. else:
  57. from hermes_cli.gateway_runtime_discovery import connect_private
  58. from gateway.control_socket import _read_response_line
  59. with connect_private(home, timeout) as peer:
  60. peer.sendall(request)
  61. def read():
  62. peer.settimeout(max(0.001, deadline - time.monotonic()))
  63. return peer.recv(65536)
  64. raw = _read_response_line(read, deadline)
  65. if not raw:
  66. raise RuntimeStoreError('runtime_draining')
  67. response = json.loads(raw)
  68. if response.get('ok') is not True:
  69. reason = str(response.get('error', '')).split(': ')[-1]
  70. if reason not in {'permission_denied', 'profile_mismatch', 'invalid_params', 'unknown_execution',
  71. 'stale_generation', 'admission_conflict', 'response_too_large', 'request_too_large'}:
  72. reason = 'runtime_draining'
  73. raise RuntimeStoreError(reason)
  74. if response.get('protocol') != 1 or response.get('id') != 1:
  75. raise RuntimeStoreError('runtime_draining')
  76. return response['result']
  77. def _attest(binding, operation, params):
  78. try:
  79. result = owner_request(binding['source_home'], 'hosted-attest', {
  80. 'selector': binding['selector'], 'operation': operation,
  81. 'params': {**params, '_target_home': binding['target_home']}})
  82. except RuntimeStoreError:
  83. raise
  84. except (OSError, ValueError) as exc:
  85. raise RuntimeStoreError('runtime_draining') from exc
  86. if not isinstance(result, dict) or not isinstance(result.get('owner'), str) or not result['owner']:
  87. raise RuntimeStoreError('permission_denied')
  88. if result.get('target_home') != binding['target_home']:
  89. raise RuntimeStoreError('permission_denied')
  90. return result
  91. def source_attachment_chunk(service, member, room_id, manifest, params):
  92. """Read scoped source bytes only on the source owner's authenticated handler.
  93. Serves one slice per call; the row's stored SHA-256 (verified at upload) rides along
  94. so the target can verify the reassembled file without the source re-hashing 15 MB
  95. per 24 KiB chunk.
  96. """
  97. index, offset = params.get('index'), params.get('offset')
  98. if (type(index) is not int or not 0 <= index < len(manifest)
  99. or type(offset) is not int or not 0 <= offset < manifest[index]['size']):
  100. raise RuntimeStoreError('permission_denied')
  101. from gateway.hosted_room_attachments import HostedRoomAttachmentStore
  102. item = manifest[index]
  103. saved = HostedRoomAttachmentStore(service.db_path).read_range(room_id=room_id,
  104. attachment_id=item['attachment_id'], event_id=item['event_id'], recipient_member_id=member,
  105. offset=offset, length=_CHUNK_BYTES)
  106. if any(saved.attachment[key] != item[key] for key in ('kind', 'name', 'mime', 'size')):
  107. raise RuntimeStoreError('permission_denied')
  108. return {'data_base64': base64.b64encode(saved.data).decode('ascii'),
  109. 'sha256': saved.attachment['sha256']}
  110. def source_attachment_digests(service, member, room_id, manifest):
  111. """Upload-verified SHA-256 of every bound input, read on the source owner's handler.
  112. Rides on the submit/execute attest result so the target can bind transferred bytes
  113. to the attested input and the preflight can verify the durable row without bytes.
  114. """
  115. from gateway.hosted_room_attachments import HostedRoomAttachmentStore
  116. store = HostedRoomAttachmentStore(service.db_path)
  117. digests = []
  118. for item in manifest:
  119. saved = store.describe(room_id=room_id, attachment_id=item['attachment_id'],
  120. event_id=item['event_id'], recipient_member_id=member)
  121. if any(saved[key] != item[key] for key in ('kind', 'name', 'mime', 'size')):
  122. raise RuntimeStoreError('permission_denied')
  123. digests.append(saved['sha256'])
  124. return digests
  125. def _attachment_data(binding, attested, params):
  126. """Transfer bytes, not foreign filenames, with a task fence on every chunk."""
  127. from gateway.hosted_room_driver import validate_bound_task_manifest
  128. manifest = attested.get('attachments', [])
  129. if not manifest:
  130. return []
  131. manifest = validate_bound_task_manifest(manifest)
  132. digests = attested.get('attachment_digests')
  133. if not isinstance(digests, list) or len(digests) != len(manifest):
  134. raise RuntimeStoreError('permission_denied')
  135. result = []
  136. for index, (item, digest) in enumerate(zip(manifest, digests)):
  137. data = bytearray()
  138. while len(data) < item['size']:
  139. chunk = _attest(binding, 'attachment', {
  140. 'task': params['task'], 'execution_generation': params['execution_generation'],
  141. 'prompt_sha256': hashlib.sha256(attested['prompt'].encode('utf-8')).hexdigest(),
  142. 'attachments': manifest, 'index': index, 'offset': len(data)})
  143. raw = base64.b64decode(chunk['data_base64'], validate=True)
  144. expected = min(_CHUNK_BYTES, item['size'] - len(data))
  145. if chunk['owner'] != attested['owner'] or len(raw) != expected or chunk['sha256'] != digest:
  146. raise RuntimeStoreError('permission_denied')
  147. data.extend(raw)
  148. if hashlib.sha256(data).hexdigest() != digest:
  149. raise RuntimeStoreError('permission_denied')
  150. result.append((item, bytes(data)))
  151. return result
  152. def _principal(authority, binding):
  153. # Source-profile namespace prevents equal room owner strings on other owners
  154. # from aliasing a target's principal. Never widen SessionActor.profile_id.
  155. identity = json.dumps([binding['source_home'], binding['owner']], separators=(',', ':'))
  156. subject = 'hosted-owner:' + hashlib.sha256(identity.encode()).hexdigest()
  157. return SessionActor(subject, authority.profile_id, _CAPS, 'hosted-owner-transport')
  158. def install_hosted_transport(server, authority, loop, *, attest):
  159. """Install profile-selected private routing over the existing authority registry.
  160. The callback must validate current room authority/member and, for submit and
  161. execute, exact TaskIdentity, generation and prompt against durable task data.
  162. It returns {'owner': durable_room_owner_subject}, never a client actor.
  163. Every wire envelope names the logical profile_id, not its control socket home.
  164. Reinstallation for a secondary preserves routing to all other served owners.
  165. """
  166. def select(envelope):
  167. from hermes_constants import assert_named_profile_home_live
  168. params = dict(envelope)
  169. profile_id = params.pop('profile_id', None)
  170. if not isinstance(profile_id, str) or not Path(profile_id).is_absolute():
  171. raise RuntimeStoreError('profile_mismatch')
  172. home = Path(profile_id)
  173. if home != home.resolve():
  174. raise RuntimeStoreError('profile_mismatch')
  175. registry = getattr(getattr(authority, 'runner', None), 'session_authorities', None)
  176. selected = registry.for_home(home) if registry is not None else authority
  177. if selected is None or selected.profile_id != profile_id:
  178. raise RuntimeStoreError('profile_mismatch')
  179. assert_named_profile_home_live(home)
  180. return selected, params
  181. def source(envelope, peer):
  182. selected, params = select(envelope)
  183. if set(params) != {'selector', 'operation', 'params'}:
  184. raise RuntimeStoreError('invalid_params')
  185. if params['operation'] not in _OPERATIONS | {'execute', 'attachment'}:
  186. raise RuntimeStoreError('invalid_params')
  187. callback = attest if selected is authority else getattr(
  188. getattr(selected, 'hosted_room_service', None), 'attest', None)
  189. if callback is None:
  190. raise RuntimeStoreError('runtime_draining')
  191. with owner_scope(selected):
  192. return callback(params['selector'], params['operation'], params['params'])
  193. def target(envelope, peer):
  194. selected, params = select(envelope)
  195. with owner_scope(selected):
  196. return produce(selected, params)
  197. def produce(authority, envelope):
  198. if set(envelope) != {'source_home', 'selector', 'operation', 'params'}:
  199. raise RuntimeStoreError('invalid_params')
  200. operation, params = envelope['operation'], dict(envelope['params'])
  201. selector = envelope['selector']
  202. if operation not in _OPERATIONS or set(selector) != {'room_id', 'member_id', 'profile'}:
  203. raise RuntimeStoreError('invalid_params')
  204. from gateway.session_authorities import served_profile_name
  205. profile = served_profile_name(Path(authority.profile_id))
  206. if selector['profile'] != profile:
  207. raise RuntimeStoreError('profile_mismatch')
  208. binding = {'source_home': envelope['source_home'], 'selector': selector,
  209. 'target_home': authority.profile_id}
  210. attested = _attest(binding, operation, params)
  211. binding['owner'] = attested['owner']
  212. principal = _principal(authority, binding)
  213. # Authorization already happened: every operation, including each attachment
  214. # chunk, is re-attested at the SOURCE owner above before anything runs here, so
  215. # the RPC's own authorize hook has nothing left to decide. A new operation must
  216. # be added to _OPERATIONS (and therefore attested) before it can reach _call.
  217. rpc = HostedRoomAuthorityRPC(authority, loop, **selector, principal=principal,
  218. authorize=lambda *args: True)
  219. key = _BINDING + rpc.ref.session_id
  220. encoded = json.dumps(binding, sort_keys=True)
  221. def persist(conn):
  222. _epoch(conn, authority.epoch)
  223. old = conn.execute('SELECT value FROM state_meta WHERE key=?', (key,)).fetchone()
  224. if old is not None and old[0] != encoded:
  225. raise RuntimeStoreError('permission_denied')
  226. conn.execute('INSERT OR IGNORE INTO state_meta(key,value) VALUES(?,?)', (key, encoded))
  227. authority.db._execute_write(persist)
  228. if operation == 'submit':
  229. rpc.hosted_attachment_data = _attachment_data(binding, attested, params)
  230. # Only the source's durable, attested text may become the admitted input.
  231. params.pop('prompt_sha256', None)
  232. params['prompt'] = attested['prompt']
  233. params['attachments'] = attested['attachments'] or None
  234. params['task'] = TaskIdentity(**params['task'])
  235. params['on_terminal'] = lambda value: None
  236. result = rpc._call(operation, **params)
  237. return result
  238. server.private_handlers.update({'hosted-attest': source, 'hosted-producer': target})
  239. def check_remote_hosted_admission(authority, ref, row):
  240. """Before claim, reauthorize durable target binding at its source owner.
  241. Returns False only for a non-transport session. A known transport binding
  242. fails closed on missing source, revoked membership or altered task payload.
  243. Call off the owner's event loop (the reverse RPC is synchronous).
  244. """
  245. with owner_scope(authority):
  246. return _check_remote_hosted_admission(authority, ref, row)
  247. def _check_remote_hosted_admission(authority, ref, row):
  248. with authority.db._read_ctx() as conn:
  249. stored = conn.execute('SELECT value FROM state_meta WHERE key=?',
  250. (_BINDING + ref.session_id,)).fetchone()
  251. if stored is None:
  252. return False
  253. try:
  254. binding = json.loads(stored[0])
  255. identity, generation = json.loads(row['request_id'][7:])
  256. if (not row['request_id'].startswith('hosted:')
  257. or not same_owner(row['principal_id'], _principal(authority, binding).subject)
  258. or ref.profile_id != authority.profile_id
  259. or binding.get('target_home') != authority.profile_id):
  260. raise ValueError('binding mismatch')
  261. params = {'task': identity, 'execution_generation': generation}
  262. attested = _attest(binding, 'execute', params)
  263. if attested['owner'] != binding['owner']:
  264. raise ValueError('owner changed')
  265. # Bytes are not re-transferred here: the durable row is compared against the
  266. # payload the attested prompt, manifest and source-verified digests commit to.
  267. from gateway.session_hosted_attachments import attested_submission_payload, verify_attested_documents
  268. if row['payload'] != attested_submission_payload(
  269. attested['prompt'], attested['attachments'], attested.get('attachment_digests')):
  270. raise ValueError('input changed')
  271. except (ValueError, KeyError, TypeError) as exc:
  272. raise RuntimeStoreError('permission_denied') from exc
  273. # Outside the permission_denied fold: a corrupted or missing retained document is a
  274. # storage fault of this destination, not a revoked source binding.
  275. verify_attested_documents(attested['attachments'], attested.get('attachment_digests'))
  276. return True
  277. class HostedRoomOwnerRPC(HostedRoomAuthorityRPC):
  278. """Driver-compatible producer; carries data, never a caller principal."""
  279. def __init__(self, *, home, source_home, room_id, member_id, profile):
  280. self.home = Path(home)
  281. self.member_id = member_id
  282. self.binding = {'source_home': str(Path(source_home)),
  283. 'selector': dict(room_id=room_id, member_id=member_id, profile=profile)}
  284. self.callbacks = {}
  285. self._lock = threading.Lock()
  286. self._monitor = None
  287. def _call(self, operation, **params):
  288. callback = params.pop('on_terminal', None)
  289. if operation == 'submit':
  290. # The private control request is 64 KiB, smaller than a legal room prompt.
  291. params['prompt_sha256'] = hashlib.sha256(params.pop('prompt').encode('utf-8')).hexdigest()
  292. if isinstance(params.get('task'), TaskIdentity):
  293. params['task'] = asdict(params['task'])
  294. result = owner_request(self.home, 'hosted-producer', {
  295. **self.binding, 'operation': operation, 'params': params})
  296. if operation in {'create', 'resume', 'resolve_exact'} and result is not None:
  297. from gateway.session_contract import SessionRef
  298. self.ref = SessionRef(str(self.home), result['session_id'])
  299. if operation == 'submit' and callback is not None:
  300. with self._lock:
  301. self.callbacks[result['admission_id']] = callback
  302. if self._monitor is None or not self._monitor.is_alive():
  303. self._monitor = threading.Thread(target=self._watch,
  304. args=(params['session_id'],), daemon=True)
  305. self._monitor.start()
  306. if operation in {'history', 'results'}:
  307. self._deliver(result)
  308. return result
  309. def history(self, *, profile, session_id, source):
  310. """Driver recovery reads terminal receipts; a transcript past one response line
  311. degrades to the bounded receipt rows (all the driver matches on) instead of failing."""
  312. try:
  313. return super().history(profile=profile, session_id=session_id, source=source)
  314. except RuntimeStoreError as exc:
  315. if exc.reason != 'response_too_large':
  316. raise
  317. return [{**receipt, 'role': 'assistant', 'content': receipt['text']}
  318. for receipt in self.results(profile=profile, session_id=session_id, source=source)]
  319. def _deliver(self, history):
  320. for row in history:
  321. with self._lock:
  322. callback = self.callbacks.pop(row.get('settlement_id'), None)
  323. if callback is not None:
  324. callback(row)
  325. def _watch(self, session_id):
  326. try:
  327. while True:
  328. with self._lock:
  329. if not self.callbacks:
  330. return
  331. pending = list(self.callbacks)[:256]
  332. # Per-admission receipts only: polling never re-reads the transcript.
  333. self.results(profile=self.binding['selector']['profile'],
  334. session_id=session_id, source='bot_room', admission_ids=pending)
  335. time.sleep(0.25)
  336. except (OSError, ValueError, RuntimeStoreError):
  337. # Retain callbacks/input for normal driver history recovery; no retry
  338. # admission and no assertion that a timed-out request was unaccepted.
  339. return