session_llm_oneshot.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262
  1. """``llm.oneshot`` on the shared gateway: a session-bound one-shot borrows that session's model.
  2. Main's sidecar looked the session up in its own table and lent its live agent's route
  3. (``tui_gateway/session_registry._main_runtime_from_agent``); a session-less call (project ideas)
  4. ran on the ``task`` auxiliary backend. The sidecar's table never holds an owner session, so on the
  5. shared gateway every Desktop one-shot fell through to the task backend. The owner answers it here:
  6. the resident agent's route, else the session's own frozen launch route.
  7. The model call never runs on the connection's serial dispatch: like the sidecar's long-handler
  8. pool, the reply is written by a detached task, so a slow commit message does not hold up the next
  9. request on that socket. Nor does it run on the loop's default executor, which session resume,
  10. socket setup and disconnect cleanup share across every connection and profile: each owner has its
  11. own small pool, and a call past its bound is refused by name instead of queueing without limit.
  12. """
  13. import asyncio
  14. from concurrent.futures import Future
  15. from contextvars import copy_context
  16. from functools import partial
  17. import logging
  18. import threading
  19. from gateway.session_busy_controls import authorize
  20. from hermes_state_runtime import RuntimeStoreError
  21. logger = logging.getLogger(__name__)
  22. _FIELDS = frozenset({'session_id', 'profile', 'template', 'instructions', 'input', 'variables', 'task',
  23. 'temperature', 'max_tokens'})
  24. _TEXT = ('template', 'instructions', 'input', 'task')
  25. # Per owner: model calls running at once, and calls admitted in all (running + queued). A call holds
  26. # its slot until its thread returns, even after its socket closed.
  27. _WORKERS = 4
  28. _ADMITTED = 16
  29. def handlers(connection):
  30. return {'llm.oneshot': partial(oneshot, connection)}
  31. class _Pool:
  32. """One owner's one-shot threads, its admitted calls, and the answer task of each per connection.
  33. Its own FIFO, not a ThreadPoolExecutor: an executor keeps a cancelled queued call (its params and
  34. copied context) until a worker dequeues it, so with every worker parked on the model a reconnect
  35. loop would pile up payloads past the bound. Here a cancelled call leaves the queue at once."""
  36. def __init__(self):
  37. self.lock = threading.Lock() # ``calls``, ``queue`` and ``threads`` are shared with the workers
  38. self.calls = set() # admitted calls, queued or running: at most _ADMITTED
  39. self.queue = {} # queued call -> its work, oldest first
  40. self.threads = 0 # worker threads alive: at most _WORKERS
  41. self.answers = {} # answer task -> connection
  42. self.closed = False
  43. self.stopping = asyncio.Event() # set on the loop by ``stop``: each answer refuses as draining
  44. def submit(self, work):
  45. """Admit ``work`` (or refuse it: ``runtime_draining``, ``oneshot_busy``) and return its Future."""
  46. call = Future()
  47. with self.lock:
  48. if self.closed:
  49. raise RuntimeStoreError('runtime_draining')
  50. if len(self.calls) >= _ADMITTED:
  51. raise RuntimeStoreError('oneshot_busy', detail=f'{_ADMITTED} one-shot generations already in flight')
  52. self.calls.add(call)
  53. self.queue[call] = work
  54. spawn = self.threads < _WORKERS
  55. self.threads += spawn
  56. call.add_done_callback(self.release)
  57. if spawn:
  58. # Daemon: a call stuck on its model must not hold up interpreter exit (it writes no state).
  59. from agent.memory_provider import spawn_context_thread
  60. try:
  61. spawn_context_thread(self.drain, name='hermes-oneshot').start()
  62. except RuntimeError:
  63. # No new worker (thread exhaustion). A live worker drains this call; with none it
  64. # would never run, so it leaves the queue with its slot and the error goes out.
  65. with self.lock:
  66. self.threads -= 1
  67. orphaned = not self.threads
  68. if orphaned and call.cancel():
  69. raise
  70. return call
  71. def release(self, call):
  72. """The call finished, or was cancelled while queued: its slot and its queued work go now."""
  73. with self.lock:
  74. self.calls.discard(call)
  75. self.queue.pop(call, None)
  76. def drain(self):
  77. while True:
  78. with self.lock:
  79. if not self.queue:
  80. self.threads -= 1
  81. return
  82. call = next(iter(self.queue))
  83. work = self.queue.pop(call)
  84. if call.set_running_or_notify_cancel():
  85. try:
  86. call.set_result(work())
  87. except BaseException as exc: # health: allow BLE001 -- the Future hands every failure to _answer, which logs it with exc_info
  88. call.set_exception(exc)
  89. del call, work # a finished call's payload is not kept while this thread waits for more
  90. def _pool(authority) -> _Pool:
  91. pool = getattr(authority, '_oneshot_pool', None)
  92. if not isinstance(pool, _Pool):
  93. pool = authority._oneshot_pool = _Pool()
  94. return pool
  95. def forget(connection):
  96. """The socket closed: drop its queued calls and answers. A call already running in its thread
  97. cannot be interrupted; it finishes, keeps its slot until then, and its reply is discarded."""
  98. pool = getattr(connection.authority, '_oneshot_pool', None)
  99. for task, owner in list(pool.answers.items()) if isinstance(pool, _Pool) else ():
  100. if owner is connection:
  101. task.cancel()
  102. def stop(authority):
  103. """Owner shutdown (whole runtime or one profile's unserve), on the loop: refuse new calls, cancel
  104. every queued call, and answer each outstanding request once with ``runtime_draining``. One
  105. profile's unserve leaves the host socket open for the others, so unlike ``forget`` its requests
  106. still get a reply. Running calls finish on their own (no SessionDB writes, only the model reply)
  107. and that reply is dropped."""
  108. pool = _pool(authority) # an owner that never served one still refuses from now on
  109. with pool.lock:
  110. pool.closed = True
  111. queued = list(pool.queue)
  112. pool.stopping.set()
  113. for call in queued:
  114. call.cancel()
  115. class DeferredReply:
  116. """A handler result answered later: ``dispatch`` hands over the request id and sends no frame."""
  117. def __init__(self, work):
  118. self.work = work # blocking callable -> result dict; runs on the owner's one-shot pool
  119. def start(self, connection, rid, check=None):
  120. """Admit the call (or refuse it: ``oneshot_busy``, ``runtime_draining``) and answer it later;
  121. ``check`` is the contract's result check, run before the reply is written."""
  122. pool = _pool(connection.authority)
  123. # The handler's context (profile scope, secret scope) carries into the thread.
  124. call = pool.submit(partial(copy_context().run, self.work))
  125. task = asyncio.get_running_loop().create_task(_answer(pool, connection.transport, rid, call, check))
  126. pool.answers[task] = connection
  127. def settled(done):
  128. pool.answers.pop(done, None)
  129. if done.cancelled():
  130. call.cancel() # still queued: leaves the queue, never runs (also before its first step)
  131. task.add_done_callback(settled)
  132. async def _answer(pool, transport, rid, call, check):
  133. """Write the request's one reply: the call's result or failure, or ``runtime_draining`` once the
  134. owner stops first. Its socket closing cancels this task instead, and then no frame goes out."""
  135. reply = asyncio.wrap_future(call)
  136. stopping = asyncio.ensure_future(pool.stopping.wait())
  137. try:
  138. await asyncio.wait((reply, stopping), return_when=asyncio.FIRST_COMPLETED)
  139. except asyncio.CancelledError: # the socket closed (``forget``): nobody reads a reply
  140. reply.cancel()
  141. raise
  142. finally:
  143. stopping.cancel()
  144. if pool.stopping.is_set() or not reply.done() or reply.cancelled():
  145. # The owner stopped first: a queued call was cancelled, a running call's late result (also one
  146. # that arrived while this task waited to wake) is dropped, its failure consumed unlogged.
  147. if not reply.cancel() and not reply.cancelled():
  148. reply.exception()
  149. frame = {'jsonrpc': '2.0', 'id': rid, 'error': {
  150. 'code': 4001, 'message': 'runtime_draining', 'data': {'reason': 'runtime_draining'}}}
  151. else:
  152. try:
  153. result = reply.result()
  154. if check is not None:
  155. check(result)
  156. frame = {'jsonrpc': '2.0', 'id': rid, 'result': result}
  157. except Exception as exc: # the request's only answer: every failure must reach the client
  158. # The sidecar's own failure code and text (tui_gateway/methods_session.py).
  159. logger.warning('llm.oneshot failed: %s', exc, exc_info=True)
  160. frame = {'jsonrpc': '2.0', 'id': rid,
  161. 'error': {'code': 5030, 'message': f'one-shot generation failed: {exc}'}}
  162. await transport.write_async(frame)
  163. def _validate(params):
  164. from agent.oneshot import PROMPT_TEMPLATES
  165. if (set(params) - _FIELDS or any(key in params and params[key] is not None and not isinstance(params[key], str)
  166. for key in (*_TEXT, 'session_id'))
  167. or not isinstance(params.get('variables') or {}, dict)
  168. or any(key in params and params[key] is not None
  169. and (isinstance(params[key], bool) or not isinstance(params[key], (int, float)))
  170. for key in ('temperature', 'max_tokens'))):
  171. raise RuntimeStoreError('invalid_params')
  172. template = (params.get('template') or '').strip()
  173. if template and template not in PROMPT_TEMPLATES:
  174. raise RuntimeStoreError('invalid_params', field='template', detail='unknown one-shot template')
  175. if not template and not (params.get('instructions') or '').strip() and not (params.get('input') or '').strip():
  176. raise RuntimeStoreError('invalid_params', field='input', detail='needs a template or instructions/input')
  177. def _session_runtime(authority, ref):
  178. """``main_runtime`` for the auxiliary client: the resident agent's live route (main's sidecar),
  179. snapshotted here on the loop, else the session's own frozen launch route as its next turn
  180. resolves it (read from config in the worker)."""
  181. from tui_gateway.session_registry import _main_runtime_from_agent
  182. agent = authority.agent(ref)
  183. if agent is not None:
  184. runtime = _main_runtime_from_agent(agent)
  185. return lambda: runtime
  186. from gateway.session_policy import policy_for_source
  187. policy = policy_for_source(authority.runner, authority.sessions[ref.session_id].source)
  188. if policy is None:
  189. return lambda: None
  190. def resolve():
  191. from gateway.run_turn_prepare import _resolve_policy_agent_runtime
  192. # fallback=False: only the session's primary route. If it cannot resolve (an AuthError), the
  193. # one-shot fails with 5030 instead of walking the route's fallback chain, because that walk
  194. # sets the runner-wide fallback notice the next turn would report.
  195. model, kwargs = _resolve_policy_agent_runtime(authority.runner, policy, fallback=False)
  196. runtime = {key: kwargs[key] for key in ('provider', 'base_url', 'api_key', 'api_mode') if kwargs.get(key)}
  197. return {**runtime, 'model': model} if model else runtime or None
  198. return resolve
  199. async def oneshot(connection, ref, params):
  200. _validate(params)
  201. authority = connection.authority
  202. if ref.session_id:
  203. authorize(connection, ref, params, 'session:read')
  204. runtime = _session_runtime(authority, ref)
  205. else:
  206. from gateway.session_authorities import require_serving
  207. require_serving(authority)
  208. if 'session:read' not in connection.actor.capabilities:
  209. raise RuntimeStoreError('permission_denied')
  210. if params.get('profile'):
  211. from pathlib import Path
  212. from hermes_cli.profiles import profile_matches_home
  213. if not profile_matches_home(params['profile'], Path(authority.profile_id)):
  214. raise RuntimeStoreError('profile_mismatch')
  215. runtime = lambda: None # the task backend
  216. def work():
  217. from agent.oneshot import run_oneshot
  218. from gateway.session_authorities import owner_scope
  219. with owner_scope(authority):
  220. return {'text': run_oneshot(
  221. instructions=params.get('instructions') or '', user_input=params.get('input') or '',
  222. template=(params.get('template') or '').strip() or None, variables=params.get('variables') or {},
  223. task=(params.get('task') or '').strip() or 'title_generation',
  224. max_tokens=int(params.get('max_tokens') or 1024),
  225. temperature=0.3 if params.get('temperature') is None else float(params['temperature']),
  226. main_runtime=runtime())}
  227. return DeferredReply(work)