session_managed_worker.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741
  1. """Admission-owned production exec; observers never own the worker lifetime."""
  2. import asyncio
  3. from contextlib import contextmanager, suppress
  4. from collections import deque
  5. from dataclasses import asdict, replace
  6. from functools import partial
  7. import json
  8. import os
  9. import queue
  10. import threading
  11. from types import SimpleNamespace
  12. from pathlib import Path
  13. import subprocess
  14. import sys
  15. import time
  16. from agent.managed_worker import accept_result, encode_frame, read_frame
  17. from gateway.session_worker_reservation import owner_worker_subject, reserve_admission_worker
  18. from hermes_state_runtime import RuntimeStoreError
  19. # The interpreter only imports psutil before it introduces itself; a child silent this long
  20. # is wedged (loader/stdio stall), not slow.
  21. HELLO_SECONDS = 60
  22. # After Stop the child interrupts its agent and emits its result; the owner terminates a
  23. # child that has not acknowledged within this window instead of waiting on the pipe forever.
  24. STOP_ACK_SECONDS = 30
  25. # How long a lost attempt's board note may hold up settling its admission (the note is
  26. # advisory; a busy board's write finishes in the background).
  27. LOSS_RECORD_WAIT_S = 10
  28. # Own session (POSIX): a worker whose owner died kills its whole group (managed_worker
  29. # _die_with_owner); on Windows it exits itself.
  30. # detached-spawn: ok — covered: the worker kills its own group when its owner dies (_die_with_owner)
  31. WORKER_POPEN = {'start_new_session': True, 'close_fds': True}
  32. class ManagedExecutionUnknown(RuntimeError):
  33. """The committed unknown verdict stops the FIFO without forging failed settlement."""
  34. def managed_policy(authority, ref):
  35. """Bypass (safe / config-only) sessions always execute out of process; other local
  36. sessions only under the explicit custom-provider opt-in.
  37. The frozen creation snapshot, not current profile config, chooses execution.
  38. """
  39. from gateway.session_policy import policy_for_source
  40. policy = policy_for_source(authority.runner, authority.sessions[ref.session_id].source)
  41. if policy is None:
  42. return None
  43. if policy.ignore_user_config or policy.kanban_json is not None:
  44. return policy
  45. if policy.config().get('gateway', {}).get('managed_workers') is not True:
  46. return None
  47. request = json.loads(policy.request_json)
  48. if policy.platform != 'cli' or request.get('provider') != 'custom' or not request.get('base_url'):
  49. raise RuntimeStoreError('unsupported_managed_policy')
  50. return policy
  51. def _bootstrap(authority, ref, row, policy, scope, text=None):
  52. from gateway.session_ingress import row_turn_author
  53. from gateway.session_policy import route_launch_key
  54. from gateway.session_policy_credentials import recover_config_secrets
  55. terminal = json.loads(policy.terminal_json)
  56. if policy.config_secret_ref:
  57. for path, value in recover_config_secrets(authority, policy).items():
  58. if path[0] is None:
  59. terminal[path[1]] = value
  60. live = authority.sessions[ref.session_id]
  61. # This turn's facts ride the per-turn hydrated request (the bootstrap field set is closed):
  62. # the admission's one-shot flags, the route's YOLO as of now (never the frozen launch flag) and
  63. # the committed surface (HUD / live voice / voice turn) when admission recorded one.
  64. request = dict(json.loads(policy.request_json), turn_v1={
  65. 'finite': row['payload'].get('finite', False), 'unattended': row['payload'].get('unattended') is True,
  66. 'yolo': _session_yolo(authority, live.route, policy),
  67. **({'surface_v1': row['payload']['surface_v1']} if row['payload'].get('surface_v1') else {}),
  68. **({'display_v1': row['payload']['display_v1']} if row['payload'].get('display_v1') else {})})
  69. from gateway.session_worker_construct import construct_inputs
  70. request['construct_v1'] = construct_inputs(authority, policy, live.source.chat_id)
  71. config = _worker_config(authority, policy)
  72. hydrated = replace(policy, config_json=json.dumps(config), request_json=json.dumps(request),
  73. terminal_json=json.dumps(terminal), credential_ref=None, config_secret_ref=None)
  74. return {'version': 1, 'home': authority.profile_id, 'scope': scope,
  75. 'policy': asdict(hydrated), 'api_key': route_launch_key(authority, policy, config),
  76. 'text': row['payload']['text'] if text is None else text, 'route': live.route,
  77. **({'attachments_v1': row['payload']['attachments_v1']} if 'attachments_v1' in row['payload'] else {}),
  78. 'user_id': live.source.user_id, 'chat_id': live.source.chat_id,
  79. 'turn_author': row_turn_author(policy, row),
  80. 'safe_mode': policy.safe_mode, 'ignore_user_config': policy.ignore_user_config}
  81. def _worker_config(authority, policy):
  82. """The frozen session config the child binds as its only config (``bind_worker_policy``).
  83. ``command_allowlist`` is approval state rather than launch policy: an ``always`` grant an
  84. earlier worker of an ordinary session persisted, or one the operator removed by hand, comes
  85. from the live profile file, as it did before the child's config was frozen. Bypass sessions
  86. never read the profile."""
  87. config = policy.config(authority)
  88. home = Path(str(authority.profile_id))
  89. if policy.ignore_user_config or not home.is_absolute():
  90. return config
  91. from gateway.run import _load_gateway_config
  92. live = _load_gateway_config(home / 'config.yaml')
  93. config.pop('command_allowlist', None)
  94. if live.get('command_allowlist') is not None:
  95. config['command_allowlist'] = live['command_allowlist']
  96. return config
  97. def _session_yolo(authority, route, policy):
  98. """The route's bypass as the in-process turn arms it on the owner, the one place a revocation
  99. is recorded: a ``--yolo`` launch seeded once per boundary, then the persisted ``/yolo`` copy."""
  100. from tools.approval import is_session_yolo_enabled
  101. from tools.approval_yolo import restore_gateway_yolo
  102. store = getattr(authority.runner, 'session_store', None)
  103. entry = store.lookup_by_session_key(route) if store is not None else None
  104. restore_gateway_yolo(route, getattr(entry, 'yolo', None), launch=bool(policy.yolo))
  105. return is_session_yolo_enabled(route)
  106. @contextmanager
  107. def worker_turn_scope(frame):
  108. """Child side of ``turn_v1``: bind what the in-process turn binds on the owner
  109. (execute_finite_admission, the route's YOLO, the committed surface), so ``chat -q``/``-z``
  110. never park a prompt, the session's current YOLO governs this child and a HUD / live-voice /
  111. voice turn reaches the model as it does in process. Frames without it bind nothing."""
  112. from gateway.session_finite import finite_turn_scope
  113. from gateway.session_display import display_turn_scope, restore_display
  114. from gateway.session_surface import restore_surface, surface_turn_scope
  115. turn = json.loads(frame['policy'].get('request_json') or '{}').get('turn_v1')
  116. if turn is None:
  117. yield
  118. return
  119. flags = {k: v for k, v in turn.items() if k not in ('surface_v1', 'display_v1')} if isinstance(turn, dict) else None
  120. if (flags is None or set(flags) != {'finite', 'unattended', 'yolo'}
  121. or any(type(v) is not bool for v in flags.values()) or (turn['unattended'] and not turn['finite'])):
  122. raise ValueError('invalid_managed_worker_bootstrap')
  123. try:
  124. surface = restore_surface(turn['surface_v1']) if 'surface_v1' in turn else None
  125. display = restore_display(turn['display_v1']) if 'display_v1' in turn else None
  126. except RuntimeStoreError as exc:
  127. raise ValueError('invalid_managed_worker_bootstrap') from exc
  128. if turn['yolo']:
  129. from tools.approval import enable_session_yolo
  130. enable_session_yolo(frame['route'])
  131. with finite_turn_scope(turn['finite'], turn['unattended']), surface_turn_scope(surface), \
  132. display_turn_scope(frame['scope']['execution_id'], display):
  133. yield
  134. class ManagedWorker:
  135. def __init__(self, process):
  136. self.process = process
  137. # Verified interpreter behind the handle (a launcher trampoline may sit between).
  138. self.worker = None
  139. self.write_lock = threading.Lock()
  140. self.commands = queue.Queue(maxsize=16)
  141. self.closed = threading.Event()
  142. # Latched the moment Stop is admitted, before any pipe write: the owner's read loop
  143. # supervises it even while the child has not yet said hello or read its bootstrap.
  144. self.stop = asyncio.Event()
  145. # Monotonic instant of the FIRST Stop: one acknowledgment budget runs from here, never
  146. # renewed by the child's later output or by a repeated Stop.
  147. self.stopped_at = None
  148. # The gateway restart/shutdown reason when that stop (not a user's) signalled this worker.
  149. self.stop_cause = None
  150. self.writer = threading.Thread(target=self._write_controls, name='managed-control-writer', daemon=True)
  151. # One dedicated reader thread per worker (never the loop's shared default executor, which
  152. # parked pipe reads would exhaust). It reads one frame per demand, so the pipe keeps its
  153. # backpressure, and posts it here; a cancelled next_frame leaves the frame for the next.
  154. self.reader = None
  155. self._demand = threading.Semaphore(0)
  156. self._requested = False
  157. self._frames = deque()
  158. self._arrived = None
  159. self.stderr_drain = None
  160. # Exact secrets this worker was handed; the stderr sink scrubs them besides the patterns.
  161. self.secrets = []
  162. def _write_controls(self):
  163. try:
  164. while not self.closed.is_set():
  165. try:
  166. frame = self.commands.get(timeout=.5)
  167. except queue.Empty:
  168. continue
  169. self.send(frame)
  170. except (OSError, ValueError):
  171. self.closed.set()
  172. def _read_frames(self, loop):
  173. """Reader thread body. It owns ``stdout``: no other thread closes it under a blocked read."""
  174. try:
  175. while self._demand.acquire() and not self.closed.is_set():
  176. try:
  177. item = read_frame(self.process.stdout)
  178. except (EOFError, ValueError, OSError, RecursionError) as exc:
  179. item = exc # EOF / invalid or oversized frame / closed pipe: the owner raises it
  180. try:
  181. loop.call_soon_threadsafe(self._deliver, item)
  182. except RuntimeError:
  183. return # the owner loop has closed; nobody is left to read for
  184. if isinstance(item, Exception):
  185. return
  186. finally:
  187. self.process.stdout.close()
  188. def _deliver(self, item):
  189. self._frames.append(item)
  190. self._arrived.set()
  191. def _take(self):
  192. item = self._frames[0]
  193. if isinstance(item, Exception):
  194. raise item # stays queued: the reader has exited, every later read fails the same way
  195. self._frames.popleft()
  196. self._requested = False
  197. if not self._frames:
  198. self._arrived.clear()
  199. return item
  200. def drain_stderr(self, path):
  201. """Route the child's stderr (its tracebacks and redirected prints) into the profile's
  202. bounded, redacted, private log instead of discarding it."""
  203. from agent.memory_provider import spawn_context_thread
  204. from gateway.session_managed_worker_log import drain_worker_stderr
  205. self.stderr_drain = spawn_context_thread(drain_worker_stderr, name='managed-stderr-drain',
  206. args=(self.process.stderr, path, self.process.pid, lambda: self.secrets))
  207. self.stderr_drain.start()
  208. def control(self, frame):
  209. if self.closed.is_set():
  210. raise RuntimeStoreError('managed_worker_lost')
  211. if frame == {'type': 'stop'}:
  212. if self.stopped_at is None:
  213. self.stopped_at = time.monotonic()
  214. self.stop.set()
  215. try:
  216. self.commands.put_nowait(frame)
  217. except queue.Full as exc:
  218. raise RuntimeStoreError('worker_control_backpressure') from exc
  219. def _stop_budget(self, ack):
  220. """Seconds left of the ``ack`` window that opened at the first Stop. Pure: only
  221. ``control`` stamps ``stopped_at``, so a read before any Stop (the hello) never starts
  222. the window a later Stop must receive in full."""
  223. return self.stopped_at + ack - time.monotonic()
  224. async def next_frame(self, timeout, ack):
  225. """Read one frame. A requested Stop bounds supervision to ``ack`` seconds TOTAL from the
  226. first Stop: a child that keeps emitting valid frames is escalated on the same deadline as a
  227. silent one, instead of leaving the turn started behind it."""
  228. if self.reader is None:
  229. self._arrived = asyncio.Event()
  230. from agent.memory_provider import spawn_context_thread
  231. self.reader = spawn_context_thread(self._read_frames, name='managed-frame-reader',
  232. args=(asyncio.get_running_loop(),))
  233. self.reader.start()
  234. if not self._frames and not self._requested:
  235. self._requested = True
  236. self._demand.release()
  237. arrived = asyncio.ensure_future(self._arrived.wait())
  238. stopper = asyncio.ensure_future(self.stop.wait())
  239. try:
  240. if not self._frames and not self.stop.is_set():
  241. await asyncio.wait({arrived, stopper}, timeout=timeout, return_when=asyncio.FIRST_COMPLETED)
  242. if not self._frames and not self.stop.is_set():
  243. raise RuntimeStoreError('managed_worker_hello_timeout')
  244. if not self._frames:
  245. # Only a latched Stop gets here; its window runs from the first Stop.
  246. remaining = self._stop_budget(ack)
  247. if remaining > 0:
  248. await asyncio.wait({arrived}, timeout=remaining)
  249. if not self._frames:
  250. raise RuntimeStoreError('managed_worker_stopped')
  251. return self._take()
  252. finally:
  253. stopper.cancel()
  254. arrived.cancel()
  255. def interrupt(self):
  256. self.control({'type': 'stop'})
  257. def respond(self, kind, prompt_id, value):
  258. self.control({'type': kind, 'prompt_id': prompt_id, 'value': value})
  259. def send(self, frame):
  260. with self.write_lock:
  261. self.process.stdin.write(encode_frame(frame))
  262. self.process.stdin.flush()
  263. def _signal_worker(self, kill):
  264. """Signal the verified interpreter, not only the handle: a launcher that exec-chained
  265. or exited leaves the real worker outside the Popen's reach."""
  266. import psutil
  267. if self.worker is None or self.worker[0] == self.process.pid:
  268. return
  269. try:
  270. proc = psutil.Process(self.worker[0])
  271. from hermes_cli.process_identity import worker_birth
  272. if worker_birth(proc) == self.worker[1]:
  273. (proc.kill if kill else proc.terminate)()
  274. except psutil.Error:
  275. pass
  276. def _signal_group(self, kill):
  277. """POSIX: signal the worker's whole process group (it leads one, WORKER_POPEN), so the tool
  278. children a killed worker never reaps with _die_with_owner go too. Only while the leader is
  279. unreaped and still leads its group; otherwise (Windows, gone) the handle alone."""
  280. if os.name != 'nt' and self.process.poll() is None:
  281. import signal
  282. try:
  283. if os.getpgid(self.process.pid) == self.process.pid:
  284. os.killpg(self.process.pid, signal.SIGKILL if kill else signal.SIGTERM) # windows-footgun: ok — POSIX only
  285. return
  286. except (ProcessLookupError, PermissionError):
  287. pass
  288. (self.process.kill if kill else self.process.terminate)()
  289. def close(self):
  290. self.closed.set()
  291. self._demand.release() # a reader parked between frames exits; one mid-read ends at EOF
  292. if self.process.poll() is None:
  293. self._signal_worker(kill=False)
  294. self._signal_group(kill=False)
  295. try:
  296. self.process.wait(timeout=5)
  297. except subprocess.TimeoutExpired:
  298. self._signal_worker(kill=True)
  299. self._signal_group(kill=True)
  300. # Still unreaped after SIGKILL (a launcher's handle): the teardown below must still run.
  301. with suppress(subprocess.TimeoutExpired):
  302. self.process.wait(timeout=5)
  303. else:
  304. self._signal_worker(kill=True)
  305. if self.writer.ident is not None:
  306. self.writer.join(timeout=5)
  307. self.process.stdin.close()
  308. if self.reader is None:
  309. self.process.stdout.close()
  310. else:
  311. self.reader.join(timeout=5)
  312. if self.stderr_drain is not None:
  313. self.stderr_drain.join(timeout=5) # EOF once the child is gone; the thread closes it
  314. elif self.process.stderr is not None:
  315. self.process.stderr.close()
  316. def interrupt_managed(authority, actor, ref, generation):
  317. worker = getattr(authority, '_managed_workers', {}).get(ref.session_id)
  318. if worker is None:
  319. return False
  320. authority.authorize(actor, ref, 'session:control')
  321. authority.check_approval_generation(ref.session_id, generation)
  322. worker.control({'type': 'stop'})
  323. return True
  324. def _prompt_frame(authority, ref, row, worker, frame):
  325. live = authority.sessions[ref.session_id]
  326. controls = live.controls
  327. kind = frame.get('type')
  328. if kind == 'prompt_settled' and set(frame) == {'type', 'prompt_id'}:
  329. prompt_id = frame['prompt_id']
  330. controls.remote_responders.pop(prompt_id, None)
  331. saved = controls.pending.pop(prompt_id, None)
  332. if saved:
  333. live.event_stream.publish(ref.session_id, {'prompt_id': prompt_id,
  334. 'execution_generation': row['generation']}, event_type=saved[1]['kind'] + '.settled')
  335. return True
  336. if kind not in {'approval', 'clarify'}:
  337. return False
  338. if len(controls.remote_responders) >= 16:
  339. raise RuntimeStoreError('worker_control_backpressure')
  340. if kind == 'approval':
  341. fields = {'request_id', 'command', 'description', 'allow_session', 'allow_permanent', 'smart_denied', 'edit'}
  342. data = frame.get('data')
  343. if set(frame) != {'type', 'data'} or not isinstance(data, dict) or set(data) - fields:
  344. raise RuntimeStoreError('invalid_worker_frame')
  345. prompt_id = data.get('request_id')
  346. else:
  347. if (set(frame) != {'type', 'prompt_id', 'question', 'choices', 'multi_select'}
  348. or not isinstance(frame['question'], str) or not isinstance(frame['choices'], list)
  349. or any(not isinstance(c, str) for c in frame['choices']) or type(frame['multi_select']) is not bool):
  350. raise RuntimeStoreError('invalid_worker_frame')
  351. prompt_id = frame['prompt_id']
  352. if not isinstance(prompt_id, str) or not prompt_id or prompt_id in controls.pending:
  353. raise RuntimeStoreError('invalid_worker_frame')
  354. controls.remote_responders[prompt_id] = worker.respond
  355. if kind == 'approval':
  356. authority.register_approval(ref.session_id, row['generation'], live.route, data)
  357. else:
  358. entry = SimpleNamespace(clarify_id=prompt_id, question=frame['question'], choices=frame['choices'],
  359. multi_select=frame['multi_select'], event=threading.Event())
  360. authority.register_clarify(ref.session_id, row['generation'], entry)
  361. return True
  362. def _worker_env(authority):
  363. """Child env for the OWNING profile under multiplex: its HERMES_HOME plus its ``.env``
  364. secrets over a scrubbed base, never the launch profile's process environment (the same
  365. rule MCP stdio children and shell hooks follow) — except for the launch profile's own
  366. worker, whose scope legitimately includes its frozen launch env. Single-profile gateways
  367. inherit the process env byte-for-byte, exactly as before."""
  368. from pathlib import Path
  369. from agent.secret_scope import is_multiplex_active
  370. from tools.environments.local import _is_routed_home
  371. home = Path(str(authority.profile_id))
  372. if not home.is_absolute():
  373. return None
  374. # Keyed on the worker's OWNING profile, not only the process-wide multiplex flag: a worker for
  375. # another profile must never inherit the launch environ even when that flag reads False.
  376. routed = _is_routed_home(home)
  377. if not routed and not is_multiplex_active():
  378. return None
  379. from agent.secret_scope import build_profile_secret_scope
  380. from tools.environments.local import _scrub_credentials, build_subprocess_env, strip_launch_profile_env
  381. # The scrub removes credentials, not settings: the launch profile's TERMINAL_* policy and
  382. # its ``.env`` settings would otherwise reach the secondary's worker (cron/kanban rule).
  383. # Strip the RAW environ before the constructor injects this turn's own session/bridge context:
  384. # a launch ``.env`` name (HERMES_SESSION_ID...) must not erase a value derived for this turn.
  385. env = build_subprocess_env(base=strip_launch_profile_env(os.environ.copy(), home), scrub_secrets=True)
  386. secrets = build_profile_secret_scope(home)
  387. if routed:
  388. # Same rule as served_profile_child_env: env_passthrough / first-party carve-outs must not
  389. # forward launch-process provider credentials that no .env or source snapshot recorded.
  390. _scrub_credentials(env, inherit_credentials=False)
  391. else:
  392. secrets = {**_launch_env_only_credentials(home, env), **secrets}
  393. env.update({k: v for k, v in secrets.items() if v is not None})
  394. env['HERMES_HOME'] = str(home)
  395. from hermes_constants import apply_subprocess_home_env
  396. apply_subprocess_home_env(env)
  397. return env
  398. def _launch_env_only_credentials(home, env):
  399. """Credentials the LAUNCH profile's owner resolves from its frozen launch env
  400. (``launch_secret_scope``: systemd ``Environment=`` / ``op run`` keys with no ``.env`` line) that
  401. the scrub removed from *env* — the same mapping the owner's own ``get_secret`` reads for this
  402. profile. Credential names only: non-credential names the constructor removed on purpose
  403. (unbound session context, venv markers) stay removed, and Hermes-internal secrets
  404. (``AUXILIARY_*_API_KEY``, relay auth) never reach a child. Only the launch home's worker calls
  405. this; a routed home's worker sees its own files only."""
  406. from tools.environments.local import _scrub_credentials
  407. from tools.environments.local_env_policy import _is_hermes_internal_secret
  408. from tui_gateway.launch_profile_policy import launch_secret_scope
  409. missing = {k: v for k, v in launch_secret_scope(home).items()
  410. if k not in env and not _is_hermes_internal_secret(k)}
  411. plain = _scrub_credentials(dict(missing), inherit_credentials=False)
  412. return {k: v for k, v in missing.items() if k not in plain}
  413. def _interrupted_before_bootstrap(authority, row, cause=None):
  414. authority.pending_results[row['admission_id']] = {
  415. 'result': {'final_response': '', 'interrupted': True, **({'interrupt_message': cause} if cause else {})},
  416. 'usage': {}}
  417. return ''
  418. def _latched_stop_cause(authority, session_id, generation):
  419. """The gateway restart/shutdown cause latched for this generation before its worker existed
  420. (``run_runtime._stop_spawning_managed``), consumed once; None for a user's Stop."""
  421. latched = getattr(authority, 'pending_stop_causes', {}).pop(session_id, None)
  422. return latched[1] if latched and latched[0] == generation else None
  423. def _with_stop_cause(accepted, worker):
  424. """The wire result carries no interruption cause; the owner recorded which stop it sent."""
  425. if accepted.get('interrupted') and worker.stop_cause:
  426. return {**accepted, 'interrupt_message': worker.stop_cause}
  427. return accepted
  428. def _settled_session_usage(authority, session_id, accepted, execution_id=None):
  429. """A managed turn's ``message.complete.usage``: the worker exits with its turn, so the session-cumulative counters come from the rows its usage writes committed. A worker that compresses mid-turn ends on a child of this reservation's session; the owner's execution row records that tip (compression moves it in the same transaction), so the path from the tip back to the reservation plus the reservation's own compression ancestors is summed, and stale siblings or the conversation a branch came from never count."""
  430. import sqlite3
  431. db = authority.db
  432. try:
  433. tip = _execution_tip(db, execution_id)
  434. path = _continuation_path(db, session_id, tip) if tip and tip != session_id else None
  435. ids = list(dict.fromkeys([*_compression_ancestors(db, session_id), *(path or [session_id])]))
  436. rows = [db.get_session(sid) or {} for sid in ids]
  437. except sqlite3.Error: # usage is display-only; settlement never waits on it
  438. return {}
  439. counts = {key: sum(int(row.get(key) or 0) for row in rows) for key in (
  440. 'input_tokens', 'output_tokens', 'cache_read_tokens', 'cache_write_tokens', 'reasoning_tokens')}
  441. prompt = counts['input_tokens'] + counts['cache_read_tokens'] + counts['cache_write_tokens']
  442. return {'session_usage': {**counts, 'prompt_tokens': prompt, 'completion_tokens': counts['output_tokens'],
  443. 'total_tokens': prompt + counts['output_tokens'],
  444. 'api_calls': sum(int(row.get('api_call_count') or 0) for row in rows),
  445. 'model': str(accepted.get('model') or rows[-1].get('model') or '')}}
  446. def _execution_tip(db, execution_id):
  447. """The session this worker execution last wrote to, as the owner's ledger records it."""
  448. if not execution_id:
  449. return None
  450. with db._read_ctx() as conn:
  451. row = conn.execute('SELECT session_id FROM worker_executions WHERE execution_id=?', (execution_id,)).fetchone()
  452. return row[0] if row else None
  453. def _continuation_path(db, start, tip):
  454. """Ids from *start* to *tip* when *tip* reaches *start* through compression continuations only, else None."""
  455. path, current = [tip], tip
  456. while current != start and len(path) <= 200:
  457. row = db.get_session(current)
  458. if not row or not db._is_compression_child_row(row):
  459. return None
  460. current = row['parent_session_id']
  461. path.append(current)
  462. return path[::-1] if current == start else None
  463. def _compression_ancestors(db, session_id):
  464. """Root-first ids of the sessions *session_id* continues by compression, excluding itself."""
  465. ancestors, current = [], db.get_session(session_id)
  466. while current and db._is_compression_child_row(current) and len(ancestors) < 200:
  467. parent = current['parent_session_id']
  468. if parent in ancestors or parent == session_id:
  469. break
  470. ancestors.append(parent)
  471. current = db.get_session(parent)
  472. return ancestors[::-1]
  473. def _mark_spawning(authority, ref, row):
  474. """Mark the generation pre-worker: until the worker registers, a gateway stop has nothing to
  475. signal, so it latches for this generation instead (#134889 GW-22). Set before ``@`` expansion,
  476. which may await the network, so a Stop there is never lost."""
  477. spawning = getattr(authority, '_managed_spawning', None)
  478. if spawning is None:
  479. spawning = authority._managed_spawning = {}
  480. spawning[ref.session_id] = row.get('generation')
  481. return spawning
  482. def _stopped_before_spawn(authority, ref, row):
  483. """A Stop latched for this generation while it had no worker (a gateway stop during ``@``
  484. expansion, or a user's Stop): consume it and record the interrupted result, never spawning."""
  485. generation = row.get('generation')
  486. stops = getattr(authority, 'pending_stops', {})
  487. if generation is None or stops.get(ref.session_id) != generation:
  488. return False
  489. del stops[ref.session_id]
  490. authority._managed_spawning.pop(ref.session_id, None)
  491. _interrupted_before_bootstrap(authority, row, _latched_stop_cause(authority, ref.session_id, generation))
  492. return True
  493. def _record_lost_attempt(authority, policy, exc, reason, scope):
  494. """Start the board disposition (``worker_lost``) for a lost Kanban card attempt; the task, or None.
  495. A worker that dies in ``worker.adopt`` never writes ``worker_bound``/``worker_result``, so
  496. without it the card's run learns nothing from its own loss. A deliberate Stop or a
  497. superseded generation is not a loss. Tracked, so retirement joins the write.
  498. """
  499. if (not getattr(policy, 'kanban_json', None) or isinstance(exc, asyncio.CancelledError)
  500. or reason in ('managed_worker_stopped', 'stale_generation')):
  501. return None
  502. from gateway.session_kanban import record_worker_loss
  503. from gateway.session_runtime_workers import track_mutation
  504. return track_mutation(authority, asyncio.to_thread(partial(record_worker_loss, policy, reason,
  505. pid=(scope or {}).get('pid'), bound=scope is not None)))
  506. async def _settle_lost(authority, row, scope, recording):
  507. """Settle a lost worker's admission ``unknown`` once its board note (``recording``) landed.
  508. The note comes first: the submitter exits on the settled 'unknown' and the crash sweep then
  509. books the bound interpreter, so a later write is dropped and an exit-0 worker_result reads
  510. as a protocol violation. A busy board holds this up ``LOSS_RECORD_WAIT_S`` at most (the
  511. write carries on, tracked), and a cancellation during the wait still settles.
  512. """
  513. from gateway.session_worker_reservation import lose_admission_worker
  514. from gateway.session_runtime_workers import tracked_write
  515. try:
  516. if recording is not None:
  517. await asyncio.wait({recording}, timeout=LOSS_RECORD_WAIT_S)
  518. finally:
  519. await tracked_write(authority, partial(lose_admission_worker, authority, row, scope))
  520. async def _spawn_registered_worker(authority, ref, row):
  521. """Spawn the turn's child and register it for Stops: ``(worker, process)``, or None when the spawn was cancelled (the
  522. interrupted result is already recorded). The generation stays marked spawning until registration."""
  523. spawning = _mark_spawning(authority, ref, row)
  524. generation = row.get('generation')
  525. try:
  526. env = await asyncio.to_thread(_worker_env, authority)
  527. cwd = (await asyncio.to_thread(Path(__file__).resolve)).parents[1]
  528. from gateway.session_worker_spawn import acquire_process
  529. from gateway.session_managed_worker_log import worker_log_path
  530. log_path = worker_log_path(authority.profile_id)
  531. process = await acquire_process(subprocess.Popen, [sys.executable, '-m', 'agent.managed_worker'],
  532. cwd=cwd, stdin=subprocess.PIPE, env=env, stdout=subprocess.PIPE,
  533. stderr=subprocess.DEVNULL if log_path is None else subprocess.PIPE, **WORKER_POPEN)
  534. except asyncio.CancelledError:
  535. # acquire_process owns late-child cleanup; no bootstrap or reservation was sent.
  536. spawning.pop(ref.session_id, None)
  537. _interrupted_before_bootstrap(authority, row, _latched_stop_cause(authority, ref.session_id, generation))
  538. return None
  539. except BaseException:
  540. spawning.pop(ref.session_id, None)
  541. raise
  542. worker = ManagedWorker(process)
  543. if log_path is not None:
  544. worker.drain_stderr(log_path)
  545. workers = getattr(authority, '_managed_workers', None)
  546. if workers is None:
  547. workers = authority._managed_workers = {}
  548. workers[ref.session_id] = worker
  549. spawning.pop(ref.session_id, None)
  550. worker.stop_cause = _latched_stop_cause(authority, ref.session_id, generation)
  551. return worker, process
  552. async def admit_managed(authority, ref, row, policy):
  553. """The ingress route into a managed worker. A removed folder is refused before a worker exists,
  554. like the in-process runner: a worker failing in its policy scope reads as a LOST execution
  555. (``unknown``) and parks every follower behind it."""
  556. from gateway.session_policy import workspace_ready, workspace_unavailable_result
  557. if not workspace_ready(policy):
  558. result = workspace_unavailable_result(policy)
  559. authority.pending_results[row['admission_id']] = {'result': result, 'usage': {}}
  560. return result['final_response']
  561. return await execute_managed(authority, ref, row, policy)
  562. async def execute_managed(authority, ref, row, policy):
  563. from gateway.run_turn_progress import publish_worker_tool_event
  564. # This route never passes the inbound prepare step, so the worker would read ``@`` tokens as typed.
  565. from gateway.session_context_refs import managed_turn_text, refused_turn_result
  566. _mark_spawning(authority, ref, row)
  567. try:
  568. text, refusal = await managed_turn_text(authority, row, policy)
  569. except BaseException:
  570. authority._managed_spawning.pop(ref.session_id, None)
  571. raise
  572. if refusal is not None:
  573. authority._managed_spawning.pop(ref.session_id, None)
  574. authority.pending_results[row['admission_id']] = {'result': refused_turn_result(row, refusal), 'usage': {}}
  575. return refusal
  576. if _stopped_before_spawn(authority, ref, row):
  577. return ''
  578. spawned = await _spawn_registered_worker(authority, ref, row)
  579. if spawned is None:
  580. return ''
  581. worker, process = spawned
  582. workers = authority._managed_workers
  583. # A Stop acknowledged during the env/spawn awaits found no worker and was latched for this
  584. # generation; consume it in the same step that makes the worker reachable to interrupt_managed.
  585. authority.adopt_agent(ref.session_id, row['generation'], worker)
  586. accepted = usage = None
  587. scope = None
  588. try:
  589. # The interpreter behind the handle introduces itself first; the owner verifies that
  590. # identity (alive, same birth, descends from the handle) before reserving for it.
  591. hello = await worker.next_frame(HELLO_SECONDS, ack=0)
  592. if worker.stop.is_set():
  593. # Stopped before the child could receive controls: hello raced the latch; never bootstrap.
  594. raise RuntimeStoreError('managed_worker_stopped')
  595. # psutil + BEGIN IMMEDIATE: off-loop and retirement-tracked, like admission and claim.
  596. from gateway.session_runtime_workers import tracked_write
  597. scope = await tracked_write(authority, partial(reserve_admission_worker, authority,
  598. admission_id=row['admission_id'], process=process, principal_id=row['principal_id'], hello=hello,
  599. worker_subject=owner_worker_subject()))
  600. worker.worker = (scope['pid'], scope['birth'])
  601. # The child reads nothing else until the exact reservation has committed.
  602. frame = await asyncio.to_thread(_bootstrap, authority, ref, row, policy, scope, text)
  603. worker.secrets.extend(v for v in (frame['api_key'], scope['secret']) if isinstance(v, str) and v)
  604. await asyncio.to_thread(worker.send, frame)
  605. worker.writer.start()
  606. while True:
  607. frame = await worker.next_frame(None, ack=STOP_ACK_SECONDS)
  608. # The claim's execution stamp, not a ledger read per frame (publish_execution fences the same way).
  609. if authority.executing.get(ref.session_id) != row['generation']:
  610. raise RuntimeStoreError('stale_generation')
  611. with authority.sessions[ref.session_id].event_stream.lock:
  612. if _prompt_frame(authority, ref, row, worker, frame):
  613. continue
  614. if frame == {'type': 'error', 'reason': 'managed_worker_failed'}:
  615. raise RuntimeStoreError('managed_worker_failed')
  616. kind = frame.get('type')
  617. if kind == 'ready' and set(frame) == {'type', 'pid'} and frame['pid'] == scope['pid']:
  618. continue
  619. if kind == 'delta' and set(frame) == {'type', 'text'} and isinstance(frame['text'], str):
  620. authority.publish_execution(ref.session_id, row['generation'], 'message.delta', {'text': frame['text']})
  621. continue
  622. if publish_worker_tool_event(authority, ref.session_id, row['generation'], frame):
  623. continue
  624. if kind == 'result' and set(frame) == {'type', 'result'} and accepted is None:
  625. try:
  626. accepted, usage = accept_result(frame['result'])
  627. except ValueError as exc:
  628. raise RuntimeStoreError('invalid_worker_result') from exc
  629. authority.sessions[ref.session_id].controls.snapshot(ref.session_id, None)
  630. await asyncio.to_thread(worker.send, {'type': 'finish'})
  631. continue
  632. if frame == {'type': 'finished'} and accepted is not None:
  633. # 'finished' follows the worker's durable execution.finish: the turn is complete.
  634. # A slow or non-zero interpreter exit after it is close()'s cleanup, not a loss.
  635. with suppress(subprocess.TimeoutExpired):
  636. await asyncio.to_thread(process.wait, 10)
  637. accepted = _with_stop_cause(accepted, worker)
  638. # Like in-process execution, settlement belongs to the drain's stream lock.
  639. authority.pending_results[row['admission_id']] = {
  640. 'result': {**accepted, 'local_viewer': True}, 'usage': usage,
  641. **await asyncio.to_thread(_settled_session_usage, authority, scope['session_id'], accepted, scope['execution_id'])}
  642. return accepted['final_response']
  643. raise RuntimeStoreError('invalid_worker_frame')
  644. except (Exception, asyncio.CancelledError) as exc:
  645. import logging
  646. reason = exc.reason if isinstance(exc, RuntimeStoreError) else type(exc).__name__
  647. logging.getLogger(__name__).warning('Managed worker lost: %s', reason)
  648. if scope is None:
  649. if isinstance(exc, asyncio.CancelledError) or (
  650. isinstance(exc, RuntimeStoreError) and exc.reason == 'managed_worker_stopped'):
  651. # Stopped before the child ever received its bootstrap: nothing executed, so
  652. # this settles like an ordinary interrupted turn (the finally kills the child).
  653. return _interrupted_before_bootstrap(authority, row, worker.stop_cause)
  654. _record_lost_attempt(authority, policy, exc, reason, scope) # nothing bound: a late note is dropped, never misbooked
  655. raise
  656. await _settle_lost(authority, row, scope, _record_lost_attempt(authority, policy, exc, reason, scope))
  657. live = authority.sessions[ref.session_id]
  658. with live.event_stream.lock:
  659. live.controls.snapshot(ref.session_id, None)
  660. authority._publish_pending(ref)
  661. live.event_stream.publish(ref.session_id, {'text': 'Worker execution is unknown.',
  662. 'content': 'Worker execution is unknown.', 'admission_id': row['admission_id'], 'outcome': 'unknown',
  663. 'status': 'error'}) # the message.complete TurnStatus: never painted as a normal finish
  664. waiter = authority.waiters.pop(row['admission_id'], None)
  665. if waiter is not None and not waiter.done():
  666. waiter.set_result('Worker execution is unknown.')
  667. # Stop this drain without its ordinary Exception→failed settlement. The
  668. # committed unknown row deliberately pauses every accepted follower.
  669. raise ManagedExecutionUnknown('managed_worker_unknown') from exc
  670. finally:
  671. workers.pop(ref.session_id, None)
  672. await asyncio.to_thread(worker.close)