terminal_approval_batch.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328
  1. """Prepare desktop terminal consent without running shells ahead of their turn.
  2. Workers keep execution middleware on its original stack. Only command approval
  3. runs ahead; the existing sequential executor releases each worker and persists
  4. its result before releasing the next. No terminal environment/cwd is acquired
  5. while preparing, and the real execution still runs every command guard.
  6. """
  7. from __future__ import annotations
  8. import contextvars
  9. import copy
  10. import threading
  11. import time
  12. from contextlib import contextmanager
  13. from typing import Any
  14. from tools.thread_context import propagate_context_to_thread
  15. _batch: contextvars.ContextVar[Any] = contextvars.ContextVar("terminal_approval_batch", default=None)
  16. _slot: contextvars.ContextVar[Any] = contextvars.ContextVar("terminal_approval_slot", default=None)
  17. class _CancelledPreparation(Exception):
  18. pass
  19. class _TerminalSlot:
  20. def __init__(self, batch, parsed, index):
  21. self.batch, self.parsed, self.index = batch, parsed, index
  22. self.ready = threading.Event()
  23. self.release = threading.Event()
  24. self.future: Any = None
  25. self.tids = []
  26. self.preparing = False
  27. self.args = None
  28. self.decision = None
  29. self.guard_key = None
  30. self.claimed = False
  31. def check_cancelled(self):
  32. if self.batch.cancelled.is_set() or self.batch.agent._interrupt_requested:
  33. raise _CancelledPreparation("Terminal approval preparation cancelled; command was not started")
  34. def prepare(self, ref):
  35. from tools import terminal_tool as tt
  36. self.check_cancelled()
  37. self.args = copy.deepcopy(ref.args)
  38. self.preparing = True
  39. try:
  40. # Read policy only. _plan_execution/_acquire_env resolve cwd and
  41. # shell state later, after the previous result has been persisted.
  42. config = tt._get_env_config()
  43. if isinstance(ref.args.get("command"), str):
  44. from tools.approval_context import set_current_observability_context, reset_current_observability_context
  45. tokens = set_current_observability_context(
  46. tool_call_id=ref.call_id, session_id=self.batch.agent.session_id or "",
  47. turn_id=getattr(self.batch.agent, "_current_turn_id", "") or "",
  48. )
  49. try:
  50. # Key on what will actually run: a foreground `cmd &` executes as `cmd`.
  51. command = ref.args["command"]
  52. if not ref.args.get("background"):
  53. command = tt.trailing_amp_background_command(command) or command
  54. self.guard_key = (command, config["env_type"], tt._docker_has_host_access(config))
  55. self.decision = tt._check_all_guards(*self.guard_key)
  56. finally:
  57. reset_current_observability_context(tokens)
  58. finally:
  59. self.preparing = False
  60. self.ready.set()
  61. while not self.release.wait(0.1):
  62. self.check_cancelled()
  63. self.check_cancelled()
  64. def run(self):
  65. from agent import tool_executor as te
  66. token = _slot.set(self)
  67. pc, batch = self.parsed, self.batch
  68. ref = pc.ref(batch.task_id)
  69. try:
  70. with te._registered_tool_worker(batch.agent) as tid:
  71. self.tids.append(tid)
  72. self.check_cancelled()
  73. dispatch = te._resolve_sequential_dispatch(batch.agent, ref, batch.messages)
  74. return te._run_agent_tool_execution_middleware(
  75. batch.agent, **ref.middleware_kwargs(), execute=dispatch.execute,
  76. scope_block=pc.scope_block, display_index=self.index + 1,
  77. authorization_gate=batch.authorization_gate,
  78. )
  79. finally:
  80. self.ready.set()
  81. _slot.reset(token)
  82. class _TerminalBatch:
  83. def __init__(self, agent, messages, task_id, parsed):
  84. from agent.tool_executor import _ConcurrentToolAuthorizationGate
  85. from tools.daemon_pool import DaemonThreadPoolExecutor
  86. self.agent, self.messages, self.task_id = agent, messages, task_id
  87. self.cancelled = threading.Event()
  88. self.pending_approvals = [] # guarded by tools.approval._lock
  89. self.authorization_gate = _ConcurrentToolAuthorizationGate()
  90. self.executor = DaemonThreadPoolExecutor(max_workers=len(parsed))
  91. self.slots = [_TerminalSlot(self, pc, i) for i, pc in enumerate(parsed)]
  92. # Set once ANY slot in the batch has published a failed result (or a
  93. # denied/blocked one). Informed consent is per the batch state the user
  94. # SAW: after a failure, a later slot's pre-collected approval no longer
  95. # describes the world its command will run in, so it must not be
  96. # consumed — the guard re-runs live instead (#113158).
  97. self.failure_seen = False
  98. def start(self):
  99. from agent.tool_executor import _resolve_sequential_tool_timeout
  100. for slot in self.slots:
  101. slot.check_cancelled()
  102. slot.future = self.executor.submit(propagate_context_to_thread(slot.run))
  103. timeout = _resolve_sequential_tool_timeout()
  104. started = time.monotonic()
  105. baseline = self.authorization_gate.excluded_seconds()
  106. # Proceed once the worker publishes a human request OR completes
  107. # preparation. A wedged plugin must not hold the batch forever.
  108. while not slot.ready.wait(0.1):
  109. slot.check_cancelled()
  110. elapsed = time.monotonic() - started - (self.authorization_gate.excluded_seconds() - baseline)
  111. if timeout is not None and elapsed >= timeout:
  112. raise TimeoutError("Terminal approval preparation timed out; commands were not started")
  113. def close(self):
  114. from agent.tool_executor import _interrupt_worker_tids
  115. from tools import approval
  116. # Withdraw only this batch's requests, including a worker wedged in
  117. # notify_cb. Thread interrupts alone leave those requests actionable.
  118. with approval._lock:
  119. self.cancelled.set()
  120. for session_key, entry in self.pending_approvals:
  121. queue = approval._gateway_queues.get(session_key, [])
  122. if entry in queue:
  123. queue.remove(entry)
  124. entry.result = "deny"
  125. entry.event.set()
  126. if not queue:
  127. approval._gateway_queues.pop(session_key, None)
  128. self.pending_approvals.clear()
  129. for slot in self.slots:
  130. slot.release.set()
  131. if slot.future is not None and not slot.future.done():
  132. _interrupt_worker_tids(self.agent, slot.tids)
  133. slot.future.cancel()
  134. self.executor.shutdown(wait=False, cancel_futures=True)
  135. def prepare_current_terminal(ref):
  136. slot = _slot.get()
  137. if slot is not None and ref.name == "terminal":
  138. slot.prepare(ref)
  139. def bind_prepared_dispatch(dispatch):
  140. """A middleware-owned thread must not lose the batch's execution barrier."""
  141. slot = _slot.get()
  142. if slot is None:
  143. return dispatch
  144. from agent.tool_executor import _registered_tool_worker
  145. owner_tid = threading.get_ident()
  146. def tracked(*args, **kwargs):
  147. if threading.get_ident() == owner_tid:
  148. return dispatch(*args, **kwargs)
  149. with _registered_tool_worker(slot.batch.agent) as tid:
  150. slot.tids.append(tid)
  151. slot.check_cancelled()
  152. return dispatch(*args, **kwargs)
  153. invoke = propagate_context_to_thread(tracked)
  154. # Each batch slot has exactly one dispatch. Reject concurrent/replayed
  155. # continuations before entering its captured Context on another thread.
  156. lock = threading.Lock()
  157. claimed = False
  158. def once(*args, **kwargs):
  159. nonlocal claimed
  160. with lock:
  161. if claimed:
  162. raise RuntimeError("Hermes tool execution callback invoked more than once")
  163. claimed = True
  164. return invoke(*args, **kwargs)
  165. return once
  166. def take_prepared_call(call_id):
  167. batch = _batch.get()
  168. if batch is None:
  169. return None
  170. for slot in batch.slots:
  171. if slot.parsed.ref(batch.task_id).call_id == call_id and not slot.claimed:
  172. slot.claimed = True
  173. slot.check_cancelled()
  174. slot.release.set()
  175. return slot
  176. return None
  177. def approval_published():
  178. slot = _slot.get()
  179. if slot is not None:
  180. slot.ready.set()
  181. def register_prepared_approval(session_key, entry):
  182. """Called under the approval queue lock, before enqueueing the request."""
  183. slot = _slot.get()
  184. if slot is not None:
  185. slot.check_cancelled()
  186. slot.batch.pending_approvals.append((session_key, entry))
  187. def consume_prepared_guard(command, env_type, has_host_access):
  188. slot = _slot.get()
  189. if slot is None or slot.preparing:
  190. return None
  191. slot.check_cancelled()
  192. # Re-gate after an earlier slot in the same batch failed (#113158): the
  193. # user approved a batch where every command was expected to run; once one
  194. # failed, that informed consent is stale for the commands after it, so
  195. # drop the pre-made decision and let the guard run its live flow
  196. # (allowlist, human approval). Nothing is auto-denied: an explicit
  197. # human answer still wins; the prepared (often auto/policy) decision is
  198. # simply not consumed.
  199. if slot.batch.failure_seen and slot.decision is not None:
  200. slot.decision = None
  201. return None
  202. from tools.approval_context import _approval_tool_call_id
  203. if (_approval_tool_call_id.get() != slot.parsed.ref(slot.batch.task_id).call_id
  204. or slot.guard_key != (command, env_type, has_host_access)):
  205. return None
  206. decision, slot.decision = slot.decision, None # single-use, even for identical calls
  207. # Batching exists to publish the human asks together. An approval nobody answered (/yolo,
  208. # approvals.mode off, the allowlist, a clean command) is policy, and the policy in force NOW
  209. # governs: switching YOLO or "Approvals: off" off mid-batch must stop the later commands.
  210. if decision is not None and decision.get("approved") and not decision.get("user_approved"):
  211. from tools.approval_context import _get_approval_mode
  212. if not (decision.get("smart_approved") and _get_approval_mode() == "smart"):
  213. return None
  214. return decision
  215. def preparing_terminal_approval():
  216. slot = _slot.get()
  217. return slot is not None and slot.preparing
  218. def mark_batch_outcome(failed: bool) -> None:
  219. """Record that the batch's current slot published a failed result.
  220. Called by the sequential publisher AFTER a result is committed, so the
  221. flag lands only for failures the model actually sees (a wedged worker's
  222. late result never publishes). Sticky for the batch: one failure re-gates
  223. every later prepared slot (#113158); successes leave it alone — a later
  224. success must not un-stale an approval after an even earlier failure.
  225. """
  226. batch = _batch.get()
  227. if batch is not None and failed:
  228. batch.failure_seen = True
  229. def validate_prepared_terminal(args):
  230. slot = _slot.get()
  231. if slot is not None:
  232. slot.check_cancelled()
  233. # Middleware/registry coercion must not turn a prepared consent into
  234. # authority for different arguments, even with identical display text.
  235. if args != slot.args:
  236. slot.decision = None
  237. raise RuntimeError("Terminal arguments changed after approval preparation; command was not started")
  238. def terminal_approval_runs(agent, calls):
  239. """Keep nonterminal barriers, but batch adjacent terminals in mixed segments."""
  240. from itertools import groupby
  241. from agent.tool_executor import _parse_tool_call
  242. def is_terminal(call):
  243. pc = _parse_tool_call(agent, call, flatten_probe=True)
  244. return pc.name == "terminal" and pc.parse_error is None
  245. for _, run in groupby(calls, key=is_terminal):
  246. yield list(run)
  247. @contextmanager
  248. def terminal_approval_batch(agent, calls, messages, task_id):
  249. from gateway.session_context import get_session_env
  250. from tools import approval
  251. from agent.tool_executor import _parse_tool_call
  252. if (len(calls) < 2 or get_session_env("HERMES_SESSION_SOURCE") != "desktop"
  253. or approval._gateway_notify_cb(approval.get_current_session_key()) is None):
  254. yield
  255. return
  256. parsed = [_parse_tool_call(agent, call, flatten_probe=True) for call in calls]
  257. # Never prepare across a nonterminal barrier; leave mixed segments and
  258. # malformed calls with the established sequential path.
  259. ids = [pc.ref(task_id).call_id for pc in parsed]
  260. if (any(pc.name != "terminal" or pc.parse_error is not None for pc in parsed)
  261. or not all(ids) or len(set(ids)) != len(ids)):
  262. yield
  263. return
  264. batch = _TerminalBatch(agent, messages, task_id, parsed)
  265. token = _batch.set(batch)
  266. try:
  267. if not agent._interrupt_requested and not getattr(agent, "_incremental_persistence_failed", False):
  268. try:
  269. batch.start()
  270. except (_CancelledPreparation, TimeoutError) as exc:
  271. batch.close()
  272. # A cancellation means a stop is already published (or our own
  273. # close() ran): re-interrupting would overwrite the user's queued
  274. # message/redirect. A timeout is a system stop: tool_reason only, no
  275. # message (callers re-queue _interrupt_message as the user's next turn).
  276. if isinstance(exc, TimeoutError):
  277. agent.interrupt(tool_reason="terminal batch preparation timeout")
  278. # The sequential path must still persist a result for every
  279. # assistant tool call, even if preparation never finished.
  280. yield
  281. finally:
  282. batch.close()
  283. _batch.reset(token)