interrupt_control.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437
  1. """Interrupt / steer / redirect control surface for ``AIAgent``.
  2. Soft/hard interrupt requests, tool-thread interrupt propagation, pending steer/redirect queues.
  3. Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
  4. """
  5. import contextlib
  6. import logging
  7. import threading
  8. from typing import Optional
  9. from agent.interrupt_compat import request_hard_interrupt
  10. from tools.interrupt import request_yield as _request_yield
  11. from tools.interrupt import set_interrupt as _set_interrupt
  12. # Same logger name as the origin module so log records / caplog filters are unchanged.
  13. logger = logging.getLogger("run_agent")
  14. # ``interrupt()`` categories that mean a human stopped the turn. Any other ``_tool_interrupt_reason`` was
  15. # supplied by a system producer via ``tool_reason`` (watchdogs, lease loss, lifecycle cancellation) and is
  16. # attributed to it in the turn exit reason instead of being booked as a user stop (#112647).
  17. _REASON_HARD_STOP = "explicit stop requested"
  18. _REASON_NEW_MESSAGE = "user sent a new message"
  19. _REASON_USER_INTERRUPT = "user interrupt"
  20. USER_INTERRUPT_REASONS = frozenset({_REASON_HARD_STOP, _REASON_NEW_MESSAGE, _REASON_USER_INTERRUPT})
  21. # ``tool_reason`` the gateway's restart/shutdown stop passes (gateway.run._INTERRUPT_TOOL_REASON_GATEWAY_SHUTDOWN).
  22. _REASON_GATEWAY_SHUTDOWN = "gateway shutdown"
  23. def interrupt_issuer(agent) -> Optional[str]:
  24. """Slug of the system producer behind the pending interrupt, or ``None`` for a human stop."""
  25. reason = getattr(agent, "_tool_interrupt_reason", None)
  26. if not reason or reason in USER_INTERRUPT_REASONS:
  27. return None
  28. return str(reason).strip().replace(" ", "_")
  29. def interrupt_skip_wording(agent) -> str:
  30. """Reason a tool call was skipped/cancelled, in words safe for tool output.
  31. Derived from the RECORDED reason rather than the call site, so a system-initiated
  32. abort reads as one instead of asserting a user message that never existed (#130207)."""
  33. reason = getattr(agent, "_tool_interrupt_reason", None)
  34. if not reason:
  35. return "Turn interrupted"
  36. if reason in USER_INTERRUPT_REASONS:
  37. return reason.capitalize()
  38. return f"Turn aborted — {str(reason).replace('_', ' ')}"
  39. def interrupted_during_api_call_reason(agent) -> str:
  40. """Turn exit reason for an API call cut short by an interrupt (``turn_explainers`` matches the prefix)."""
  41. issuer = interrupt_issuer(agent)
  42. return f"interrupted_during_api_call({issuer})" if issuer else "interrupted_during_api_call"
  43. def _fence_cancel_before_commit(fence, *, when_in_flight: bool, failure_log: str) -> None:
  44. """Call ``type(fence).cancel_before_commit(fence)`` when ``commit_in_flight`` matches.
  45. Hard-cancel admission has two halves (#99758 P1). BEFORE the generation claim is
  46. consumed only a commit already in flight is waited out (the call blocks on the fence
  47. lock and returns False WITHOUT setting ``_cancelled``) — cancelling a still-pending
  48. fence there would be irreversible for an abort that may yet be declined. AFTER the
  49. claim survived, only a still-pending commit is cancelled; one that started meanwhile
  50. owns the fence and completes on its own."""
  51. if fence is None or bool(getattr(fence, "commit_in_flight", False)) is not when_in_flight:
  52. return
  53. cancel_before_commit = getattr(type(fence), "cancel_before_commit", None)
  54. if callable(cancel_before_commit):
  55. try:
  56. cancel_before_commit(fence)
  57. except Exception:
  58. logger.debug(failure_log, exc_info=True)
  59. def _ic_lock(agent, attr: str):
  60. """``with`` the lock stored at ``attr`` when present; __init__-less test stubs run unlocked."""
  61. lock = getattr(agent, attr, None)
  62. return contextlib.nullcontext() if lock is None else lock
  63. def _ic_slot(agent, lock_attr: str, slot: str):
  64. """Read the pending-text ``slot`` guarded by ``lock_attr``. An initialized agent always has both
  65. attributes, so under the lock the slot is read directly and a missing one fails loud (a real bug);
  66. only ``__init__``-less test stubs (no lock) get the ``getattr`` fallback."""
  67. if getattr(agent, lock_attr, None) is None:
  68. return getattr(agent, slot, None)
  69. return getattr(agent, slot)
  70. def _steer_parts(agent) -> list:
  71. """The buffered steers as ``(ticket, text)`` pairs; ``_pending_steer`` is their newline join. A ticket
  72. names one submission (a gateway transfer). A buffer another writer set directly is one ticketless part."""
  73. pending = _ic_slot(agent, "_pending_steer_lock", "_pending_steer")
  74. parts = vars(agent).get("_pending_steer_parts") or []
  75. if "\n".join(text for _, text in parts) != (pending or ""):
  76. return [(None, pending)] if pending else []
  77. return list(parts)
  78. def _set_steer_parts(agent, parts: list) -> None:
  79. agent._pending_steer_parts = parts
  80. agent._pending_steer = "\n".join(text for _, text in parts) or None
  81. def _steer_fates(agent) -> dict:
  82. """``{ticket: consumed}`` for ticketed steers that left the buffer: True when a drain handed the text
  83. to the turn, False when it was dropped unread. ``withdraw_steer`` reads each one once."""
  84. return vars(agent).setdefault("_steer_ticket_fates", {})
  85. def requeue_drained_steer(agent, steer_text: str) -> None:
  86. """Put drained steer text back (nothing took it). When it is the last drain's text, its parts go back
  87. with their tickets, so a transfer in it reads unconsumed again."""
  88. with _ic_lock(agent, "_pending_steer_lock"):
  89. drained, parts = vars(agent).pop("_steer_last_drain", (None, None))
  90. if drained != steer_text or not parts:
  91. parts = [(None, steer_text)]
  92. for ticket, _ in parts:
  93. _steer_fates(agent).pop(ticket, None)
  94. _set_steer_parts(agent, _steer_parts(agent) + parts)
  95. def _ic_codex_method(agent, name: str):
  96. """Codex app-server owns its model/tool loop; return its ``name`` hook or None."""
  97. if getattr(agent, "api_mode", None) != "codex_app_server":
  98. return None
  99. method = getattr(getattr(agent, "_codex_session", None), name, None)
  100. return method if callable(method) else None
  101. def _ic_abort_active_request(agent, reason: str, failure_log: str) -> None:
  102. """Shut the registered in-flight request's sockets (cron turns register their client here)."""
  103. abort = getattr(agent, "_active_request_abort", None)
  104. if callable(abort):
  105. try:
  106. abort(reason)
  107. except Exception:
  108. logger.debug(failure_log, exc_info=True)
  109. def _ic_signal_tool_workers(agent, active: bool, **kw) -> None:
  110. """Fan the tool interrupt bit out to concurrent-tool worker tids.
  111. ``is_interrupted()`` inside a tool only sees its own tid, so without this a hung
  112. concurrent tool runs to its own timeout (and a stale bit could survive a turn
  113. boundary onto a recycled tid). getattr covers __init__-less stubs."""
  114. tracker = getattr(agent, "_tool_worker_threads", None)
  115. tracker_lock = getattr(agent, "_tool_worker_threads_lock", None)
  116. if tracker is None or tracker_lock is None:
  117. return
  118. with tracker_lock:
  119. worker_tids = list(tracker)
  120. for tid in worker_tids:
  121. try:
  122. _set_interrupt(active, tid, **kw)
  123. except Exception:
  124. pass
  125. class InterruptControlMixin:
  126. """interrupt()/hard_interrupt()/clear_interrupt()/steer()/redirect() (see module docstring)."""
  127. def interrupt(
  128. self, message: Optional[str] = None, *, hard_cancel: bool = False,
  129. tool_reason: Optional[str] = None, require_generation: Optional[int] = None,
  130. ) -> bool:
  131. """Request the agent to interrupt its current tool-calling loop (call from another thread).
  132. ``hard_cancel``: explicit stop; compression may honor it even while ordinary interrupts are masked.
  133. ``tool_reason``: trusted fixed category safe for tool output. ``require_generation``: activity-
  134. generation claim — published only if the turn's generation still matches at the final mutation edge;
  135. returns False if the turn resumed meanwhile.
  136. """
  137. if require_generation is not None:
  138. # RESERVE the claim under the SAME lock `_touch_activity` stamps with; real progress invalidates
  139. # it and it is CONSUMED at the final mutation edge, so a resumed turn abandons the abort.
  140. with self._liveness_activity_lock():
  141. if getattr(self, "_turn_liveness_activity_generation", 0) != require_generation:
  142. return False
  143. self._turn_liveness_abort_claim = require_generation
  144. # Tool cancellation attribution stays separate from _interrupt_message, which may carry the user's
  145. # full next message. An explicit ``tool_reason`` wins on BOTH paths: a system producer that has to
  146. # stop the turn SOFTLY (batch guards, watchdogs) previously had no way to label itself — passing a
  147. # message made it look like "the user sent a new message", so the abort was attributed to the user
  148. # and rendered as a user-stop placeholder (#130207).
  149. tool_interrupt_reason = (
  150. (tool_reason or _REASON_HARD_STOP) if hard_cancel
  151. else (tool_reason or (_REASON_NEW_MESSAGE if message else _REASON_USER_INTERRUPT))
  152. )
  153. def _publish_interrupt_state() -> None:
  154. # A shutdown stop never relabels a user Stop already pending (the user's Stop wins): the turn
  155. # records the first cause, so it is not announced as interrupted by the restart. A pending
  156. # redirect (a mid-request correction) is not a Stop: the restart drops it and is the cause.
  157. if not (tool_reason == _REASON_GATEWAY_SHUTDOWN and getattr(self, "_interrupt_requested", False)
  158. and getattr(self, "_tool_interrupt_reason", None) in USER_INTERRUPT_REASONS):
  159. self._interrupt_message = message
  160. self._tool_interrupt_reason = tool_interrupt_reason
  161. self._interrupt_requested = True
  162. self._turn_user_intervened = True
  163. # The turn record and the log must agree on WHO asked for the stop (#112647).
  164. logger.info("Interrupt requested (%s): %s", "hard" if hard_cancel else "soft", tool_interrupt_reason)
  165. _hard_event = getattr(self, "_hard_interrupt_requested", None) if hard_cancel else None
  166. if _hard_event is not None:
  167. _hard_event.set()
  168. def _fence(): # re-read each time: a finished commit may replace or clear the slot
  169. return vars(self).get("_active_compression_commit_fence") if hard_cancel else None
  170. # A hard stop and redirect share one lock so /stop cannot race with an accepted correction and
  171. # accidentally turn itself into a retry. The blocking in-flight-commit wait runs BEFORE the atomic
  172. # claim edge (redirect lock still held); the destructive pending-commit cancel runs AFTER the claim
  173. # survives (#99758 P1).
  174. with _ic_lock(self, "_pending_redirect_lock"):
  175. _fence_cancel_before_commit(
  176. _fence(), when_in_flight=True, failure_log="Compression hard-cancel fence wait failed"
  177. )
  178. if require_generation is None:
  179. # No claim to race: publish WITHOUT the liveness lock (bare AIAgent stand-ins in other
  180. # suites lack the liveness seam and would AttributeError).
  181. _publish_interrupt_state()
  182. else:
  183. # Final mutation edge: claim consumption and the FIRST observable publication are ONE
  184. # activity-lock critical section, so either the claim survives and commits before any later
  185. # activity stamp, or the stamp landed first and the abort declines without publishing.
  186. with self._liveness_activity_lock():
  187. if getattr(self, "_turn_liveness_abort_claim", None) != require_generation:
  188. return False
  189. self._turn_liveness_abort_claim = None
  190. _publish_interrupt_state()
  191. _fence_cancel_before_commit(
  192. _fence(), when_in_flight=False, failure_log="Compression hard-cancel fence admission failed"
  193. )
  194. self._pending_redirect = None
  195. # Codex watches a private interrupt event rather than Hermes' per-thread flag.
  196. _request_interrupt = _ic_codex_method(self, "request_interrupt")
  197. if _request_interrupt is not None:
  198. try:
  199. _request_interrupt()
  200. except Exception:
  201. logger.debug("Failed to interrupt Codex app-server turn", exc_info=True)
  202. # Cron turns request on the conversation thread (no nested interrupt-worker deadlock); their client
  203. # is registered here so this cross-thread interrupt can still shut the sockets.
  204. _ic_abort_active_request(self, "interrupt_abort", "Failed to abort active inline request")
  205. # Scope the tool interrupt to this agent's execution thread so other in-process agents are unaffected.
  206. if self._execution_thread_id is not None:
  207. _set_interrupt(True, self._execution_thread_id, reason=tool_interrupt_reason)
  208. self._interrupt_thread_signal_pending = False
  209. else:
  210. # Interrupt arrived before run_conversation bound the execution thread: defer the tool-level
  211. # signal instead of targeting the caller thread.
  212. self._interrupt_thread_signal_pending = True
  213. _ic_signal_tool_workers(self, True, reason=tool_interrupt_reason)
  214. # Propagate interrupt to any running child agents (subagent delegation)
  215. with self._active_children_lock:
  216. children_copy = list(self._active_children)
  217. for child in children_copy:
  218. try:
  219. if hard_cancel:
  220. request_hard_interrupt(child, message, tool_reason=tool_interrupt_reason)
  221. else:
  222. child.interrupt(message)
  223. except Exception as e:
  224. logger.debug("Failed to propagate interrupt to child agent: %s", e)
  225. if not self.quiet_mode:
  226. print("\n⚡ Interrupt requested" + (f": '{message[:40]}...'" if message and len(message) > 40 else f": '{message}'" if message else ""))
  227. return True
  228. def hard_interrupt(self, message: Optional[str] = None, *, tool_reason: Optional[str] = None) -> None:
  229. """Explicit stop preserving the ``interrupt()`` ABI (frontends feature-detect this and fall back to
  230. legacy ``interrupt()`` for third-party agents). Bypasses dynamic dispatch: legacy subclasses may
  231. override interrupt(message=None) without hard_cancel."""
  232. InterruptControlMixin.interrupt(self, message, hard_cancel=True, tool_reason=tool_reason)
  233. def clear_interrupt(self, *, preserve_redirect: bool = False, hard_cancel: bool = False) -> bool:
  234. """Clear the interrupt request and per-thread tool signal. ``preserve_redirect`` is only for the
  235. conversation loop rebuilding the same logical turn after cancelling a model request.
  236. ``hard_cancel`` also drops an unconsumed pending steer: the aborted turn's next tool iteration
  237. will no longer happen, and re-injecting the note into the post-stop turn would surprise the
  238. user. Every other caller (redirect rebuild, error recovery, turn-boundary hygiene) continues
  239. this session, so the already-accepted steer must survive: it stays buffered for the existing
  240. drains — the pre-API inject, the post-batch append, or the finalizer's leftover handoff —
  241. instead of silently vanishing after the surface was told it was delivered."""
  242. with _ic_lock(self, "_pending_redirect_lock"):
  243. if preserve_redirect and not _ic_slot(self, "_pending_redirect_lock", "_pending_redirect"):
  244. return False
  245. self._interrupt_requested = False
  246. self._interrupt_message = self._tool_interrupt_reason = None
  247. getattr(self, "_hard_interrupt_requested", threading.Event()).clear()
  248. if not preserve_redirect:
  249. self._pending_redirect = None
  250. self._interrupt_thread_signal_pending = False
  251. if self._execution_thread_id is not None:
  252. _set_interrupt(False, self._execution_thread_id)
  253. _ic_signal_tool_workers(self, False)
  254. if hard_cancel:
  255. # Hard stop only (see docstring). The comment that used to run unconditionally here
  256. # claimed a hard interrupt supersedes the steer — but nothing gated this wipe on
  257. # hard_cancel, so a soft clear dropped a live user message with no trace.
  258. with _ic_lock(self, "_pending_steer_lock"):
  259. for ticket, _ in _steer_parts(self):
  260. if ticket is not None:
  261. _steer_fates(self)[ticket] = False
  262. _set_steer_parts(self, [])
  263. return True
  264. def steer(self, text: str, *, ticket=None) -> bool:
  265. """Queue user text for delivery as its own user row after the current tool batch finishes (no
  266. interrupt); multiple calls concatenate with newlines. Returns False for empty text. ``ticket``
  267. names the submission, so ``withdraw_steer`` can settle exactly it, whatever text others carry."""
  268. if not text or not text.strip():
  269. return False
  270. cleaned = text.strip()
  271. with _ic_lock(self, "_pending_steer_lock"):
  272. _set_steer_parts(self, _steer_parts(self) + [(ticket, cleaned)])
  273. # The turn no longer ran on its own (first-task-done detection reads this off the result).
  274. self._turn_user_intervened = True
  275. return True
  276. def redirect(self, text: str) -> bool:
  277. """Redirect the active turn without converting it into a new task: during a model request only that
  278. request is cancelled (completed messages kept, partial reasoning becomes assistant context, the
  279. correction is appended as a real user message, the loop retries); during tool execution it degrades
  280. to ``steer()``; Codex app-server uses native ``turn/steer``. False when no live turn / empty text."""
  281. if not text or not text.strip():
  282. return False
  283. cleaned = text.strip()
  284. _native_steer = _ic_codex_method(self, "request_steer")
  285. if _native_steer is not None:
  286. with _ic_lock(self, "_pending_redirect_lock"):
  287. if self._interrupt_requested:
  288. return False
  289. try:
  290. accepted = bool(_native_steer(cleaned))
  291. if accepted:
  292. self._turn_user_intervened = True
  293. return accepted
  294. except Exception:
  295. logger.debug("Codex app-server turn/steer failed", exc_info=True)
  296. return False
  297. # Never kill a tool to deliver guidance; the steer drain puts it on the final tool result.
  298. # A foreground terminal command would park that delivery until it exits (a 5-minute
  299. # `sleep` poller, a build), so ask the tool workers to YIELD: terminal hands the live
  300. # process to the background registry and returns; tools that don't yield are unaffected.
  301. if getattr(self, "_executing_tools", False):
  302. accepted = self.steer(cleaned)
  303. if accepted:
  304. tracker = getattr(self, "_tool_worker_threads", None)
  305. tracker_lock = getattr(self, "_tool_worker_threads_lock", None)
  306. if tracker is not None and tracker_lock is not None:
  307. with tracker_lock:
  308. worker_tids = list(tracker)
  309. for tid in worker_tids:
  310. _request_yield(tid)
  311. return accepted
  312. _model_active = getattr(self, "_model_request_active", None)
  313. with _ic_lock(self, "_pending_redirect_lock"):
  314. request_active = _model_active is not None and _model_active.is_set()
  315. if not request_active and not getattr(self, "_turn_redirect_open", False):
  316. return False
  317. existing = _ic_slot(self, "_pending_redirect_lock", "_pending_redirect")
  318. if self._interrupt_requested and not existing:
  319. return False
  320. self._pending_redirect = (
  321. f"{existing}\n\n[Additional user correction]\n{cleaned}" if existing else cleaned
  322. )
  323. self._turn_user_intervened = True
  324. if not request_active:
  325. # Preparation has no request to cancel. The iteration entry or
  326. # pre-call gate will append this correction before model use.
  327. return True
  328. self._interrupt_requested = True
  329. self._interrupt_message = None
  330. # Interrupt only the model request — no fan-out to tool workers / child agents as interrupt() does.
  331. _execution_thread_id = getattr(self, "_execution_thread_id", None)
  332. if _execution_thread_id is not None:
  333. _set_interrupt(True, _execution_thread_id)
  334. self._interrupt_thread_signal_pending = False
  335. else:
  336. self._interrupt_thread_signal_pending = True
  337. _ic_abort_active_request(self, "redirect_abort", "Failed to abort request for redirect")
  338. return True
  339. def _has_pending_redirect(self) -> bool:
  340. """Return whether an active-turn redirect is waiting to be applied."""
  341. with _ic_lock(self, "_pending_redirect_lock"):
  342. return bool(_ic_slot(self, "_pending_redirect_lock", "_pending_redirect"))
  343. def _drain_pending_redirect(self) -> Optional[str]:
  344. """Return and clear pending active-turn correction text."""
  345. with _ic_lock(self, "_pending_redirect_lock"):
  346. text = _ic_slot(self, "_pending_redirect_lock", "_pending_redirect")
  347. self._pending_redirect = None
  348. return text
  349. def _drain_pending_steer(self) -> Optional[str]:
  350. """Return the pending steer text (if any) and clear the slot; None when nothing is pending."""
  351. with _ic_lock(self, "_pending_steer_lock"):
  352. text = _ic_slot(self, "_pending_steer_lock", "_pending_steer")
  353. parts = _steer_parts(self)
  354. _set_steer_parts(self, [])
  355. for ticket, _ in parts:
  356. if ticket is not None:
  357. _steer_fates(self)[ticket] = True
  358. self._steer_last_drain = (text, parts) # requeue_drained_steer restores its tickets
  359. return text
  360. def _drain_leftover_steer(self) -> Optional[str]:
  361. """The turn finalizer's drain: return and clear the ticketless steers (they become the next user
  362. turn) and keep each ticketed one buffered. A transfer is never a follow-up a restart may discard:
  363. settlement withdraws it and its admission goes back to its own queue place."""
  364. with _ic_lock(self, "_pending_steer_lock"):
  365. parts = _steer_parts(self)
  366. _set_steer_parts(self, [part for part in parts if part[0] is not None])
  367. return "\n".join(text for ticket, text in parts if ticket is None) or None
  368. def withdraw_steer(self, ticket) -> bool:
  369. """Settle one ``steer(text, ticket=...)`` by identity (the gateway's transfer of a queued prompt):
  370. True when no drain handed it to the turn, either still buffered (taken out now; every other steer
  371. stays, whatever its text) or dropped unread; False once a drain consumed it, or for an unknown one."""
  372. if ticket is None:
  373. return False
  374. with _ic_lock(self, "_pending_steer_lock"):
  375. parts = _steer_parts(self)
  376. kept = [part for part in parts if part[0] != ticket]
  377. consumed = _steer_fates(self).pop(ticket, True)
  378. if len(kept) == len(parts):
  379. return consumed is False
  380. _set_steer_parts(self, kept)
  381. return True