| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437 |
- """Interrupt / steer / redirect control surface for ``AIAgent``.
- Soft/hard interrupt requests, tool-thread interrupt propagation, pending steer/redirect queues.
- Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
- """
- import contextlib
- import logging
- import threading
- from typing import Optional
- from agent.interrupt_compat import request_hard_interrupt
- from tools.interrupt import request_yield as _request_yield
- from tools.interrupt import set_interrupt as _set_interrupt
- # Same logger name as the origin module so log records / caplog filters are unchanged.
- logger = logging.getLogger("run_agent")
- # ``interrupt()`` categories that mean a human stopped the turn. Any other ``_tool_interrupt_reason`` was
- # supplied by a system producer via ``tool_reason`` (watchdogs, lease loss, lifecycle cancellation) and is
- # attributed to it in the turn exit reason instead of being booked as a user stop (#112647).
- _REASON_HARD_STOP = "explicit stop requested"
- _REASON_NEW_MESSAGE = "user sent a new message"
- _REASON_USER_INTERRUPT = "user interrupt"
- USER_INTERRUPT_REASONS = frozenset({_REASON_HARD_STOP, _REASON_NEW_MESSAGE, _REASON_USER_INTERRUPT})
- # ``tool_reason`` the gateway's restart/shutdown stop passes (gateway.run._INTERRUPT_TOOL_REASON_GATEWAY_SHUTDOWN).
- _REASON_GATEWAY_SHUTDOWN = "gateway shutdown"
- def interrupt_issuer(agent) -> Optional[str]:
- """Slug of the system producer behind the pending interrupt, or ``None`` for a human stop."""
- reason = getattr(agent, "_tool_interrupt_reason", None)
- if not reason or reason in USER_INTERRUPT_REASONS:
- return None
- return str(reason).strip().replace(" ", "_")
- def interrupt_skip_wording(agent) -> str:
- """Reason a tool call was skipped/cancelled, in words safe for tool output.
- Derived from the RECORDED reason rather than the call site, so a system-initiated
- abort reads as one instead of asserting a user message that never existed (#130207)."""
- reason = getattr(agent, "_tool_interrupt_reason", None)
- if not reason:
- return "Turn interrupted"
- if reason in USER_INTERRUPT_REASONS:
- return reason.capitalize()
- return f"Turn aborted — {str(reason).replace('_', ' ')}"
- def interrupted_during_api_call_reason(agent) -> str:
- """Turn exit reason for an API call cut short by an interrupt (``turn_explainers`` matches the prefix)."""
- issuer = interrupt_issuer(agent)
- return f"interrupted_during_api_call({issuer})" if issuer else "interrupted_during_api_call"
- def _fence_cancel_before_commit(fence, *, when_in_flight: bool, failure_log: str) -> None:
- """Call ``type(fence).cancel_before_commit(fence)`` when ``commit_in_flight`` matches.
- Hard-cancel admission has two halves (#99758 P1). BEFORE the generation claim is
- consumed only a commit already in flight is waited out (the call blocks on the fence
- lock and returns False WITHOUT setting ``_cancelled``) — cancelling a still-pending
- fence there would be irreversible for an abort that may yet be declined. AFTER the
- claim survived, only a still-pending commit is cancelled; one that started meanwhile
- owns the fence and completes on its own."""
- if fence is None or bool(getattr(fence, "commit_in_flight", False)) is not when_in_flight:
- return
- cancel_before_commit = getattr(type(fence), "cancel_before_commit", None)
- if callable(cancel_before_commit):
- try:
- cancel_before_commit(fence)
- except Exception:
- logger.debug(failure_log, exc_info=True)
- def _ic_lock(agent, attr: str):
- """``with`` the lock stored at ``attr`` when present; __init__-less test stubs run unlocked."""
- lock = getattr(agent, attr, None)
- return contextlib.nullcontext() if lock is None else lock
- def _ic_slot(agent, lock_attr: str, slot: str):
- """Read the pending-text ``slot`` guarded by ``lock_attr``. An initialized agent always has both
- attributes, so under the lock the slot is read directly and a missing one fails loud (a real bug);
- only ``__init__``-less test stubs (no lock) get the ``getattr`` fallback."""
- if getattr(agent, lock_attr, None) is None:
- return getattr(agent, slot, None)
- return getattr(agent, slot)
- def _steer_parts(agent) -> list:
- """The buffered steers as ``(ticket, text)`` pairs; ``_pending_steer`` is their newline join. A ticket
- names one submission (a gateway transfer). A buffer another writer set directly is one ticketless part."""
- pending = _ic_slot(agent, "_pending_steer_lock", "_pending_steer")
- parts = vars(agent).get("_pending_steer_parts") or []
- if "\n".join(text for _, text in parts) != (pending or ""):
- return [(None, pending)] if pending else []
- return list(parts)
- def _set_steer_parts(agent, parts: list) -> None:
- agent._pending_steer_parts = parts
- agent._pending_steer = "\n".join(text for _, text in parts) or None
- def _steer_fates(agent) -> dict:
- """``{ticket: consumed}`` for ticketed steers that left the buffer: True when a drain handed the text
- to the turn, False when it was dropped unread. ``withdraw_steer`` reads each one once."""
- return vars(agent).setdefault("_steer_ticket_fates", {})
- def requeue_drained_steer(agent, steer_text: str) -> None:
- """Put drained steer text back (nothing took it). When it is the last drain's text, its parts go back
- with their tickets, so a transfer in it reads unconsumed again."""
- with _ic_lock(agent, "_pending_steer_lock"):
- drained, parts = vars(agent).pop("_steer_last_drain", (None, None))
- if drained != steer_text or not parts:
- parts = [(None, steer_text)]
- for ticket, _ in parts:
- _steer_fates(agent).pop(ticket, None)
- _set_steer_parts(agent, _steer_parts(agent) + parts)
- def _ic_codex_method(agent, name: str):
- """Codex app-server owns its model/tool loop; return its ``name`` hook or None."""
- if getattr(agent, "api_mode", None) != "codex_app_server":
- return None
- method = getattr(getattr(agent, "_codex_session", None), name, None)
- return method if callable(method) else None
- def _ic_abort_active_request(agent, reason: str, failure_log: str) -> None:
- """Shut the registered in-flight request's sockets (cron turns register their client here)."""
- abort = getattr(agent, "_active_request_abort", None)
- if callable(abort):
- try:
- abort(reason)
- except Exception:
- logger.debug(failure_log, exc_info=True)
- def _ic_signal_tool_workers(agent, active: bool, **kw) -> None:
- """Fan the tool interrupt bit out to concurrent-tool worker tids.
- ``is_interrupted()`` inside a tool only sees its own tid, so without this a hung
- concurrent tool runs to its own timeout (and a stale bit could survive a turn
- boundary onto a recycled tid). getattr covers __init__-less stubs."""
- tracker = getattr(agent, "_tool_worker_threads", None)
- tracker_lock = getattr(agent, "_tool_worker_threads_lock", None)
- if tracker is None or tracker_lock is None:
- return
- with tracker_lock:
- worker_tids = list(tracker)
- for tid in worker_tids:
- try:
- _set_interrupt(active, tid, **kw)
- except Exception:
- pass
- class InterruptControlMixin:
- """interrupt()/hard_interrupt()/clear_interrupt()/steer()/redirect() (see module docstring)."""
- def interrupt(
- self, message: Optional[str] = None, *, hard_cancel: bool = False,
- tool_reason: Optional[str] = None, require_generation: Optional[int] = None,
- ) -> bool:
- """Request the agent to interrupt its current tool-calling loop (call from another thread).
- ``hard_cancel``: explicit stop; compression may honor it even while ordinary interrupts are masked.
- ``tool_reason``: trusted fixed category safe for tool output. ``require_generation``: activity-
- generation claim — published only if the turn's generation still matches at the final mutation edge;
- returns False if the turn resumed meanwhile.
- """
- if require_generation is not None:
- # RESERVE the claim under the SAME lock `_touch_activity` stamps with; real progress invalidates
- # it and it is CONSUMED at the final mutation edge, so a resumed turn abandons the abort.
- with self._liveness_activity_lock():
- if getattr(self, "_turn_liveness_activity_generation", 0) != require_generation:
- return False
- self._turn_liveness_abort_claim = require_generation
- # Tool cancellation attribution stays separate from _interrupt_message, which may carry the user's
- # full next message. An explicit ``tool_reason`` wins on BOTH paths: a system producer that has to
- # stop the turn SOFTLY (batch guards, watchdogs) previously had no way to label itself — passing a
- # message made it look like "the user sent a new message", so the abort was attributed to the user
- # and rendered as a user-stop placeholder (#130207).
- tool_interrupt_reason = (
- (tool_reason or _REASON_HARD_STOP) if hard_cancel
- else (tool_reason or (_REASON_NEW_MESSAGE if message else _REASON_USER_INTERRUPT))
- )
- def _publish_interrupt_state() -> None:
- # A shutdown stop never relabels a user Stop already pending (the user's Stop wins): the turn
- # records the first cause, so it is not announced as interrupted by the restart. A pending
- # redirect (a mid-request correction) is not a Stop: the restart drops it and is the cause.
- if not (tool_reason == _REASON_GATEWAY_SHUTDOWN and getattr(self, "_interrupt_requested", False)
- and getattr(self, "_tool_interrupt_reason", None) in USER_INTERRUPT_REASONS):
- self._interrupt_message = message
- self._tool_interrupt_reason = tool_interrupt_reason
- self._interrupt_requested = True
- self._turn_user_intervened = True
- # The turn record and the log must agree on WHO asked for the stop (#112647).
- logger.info("Interrupt requested (%s): %s", "hard" if hard_cancel else "soft", tool_interrupt_reason)
- _hard_event = getattr(self, "_hard_interrupt_requested", None) if hard_cancel else None
- if _hard_event is not None:
- _hard_event.set()
- def _fence(): # re-read each time: a finished commit may replace or clear the slot
- return vars(self).get("_active_compression_commit_fence") if hard_cancel else None
- # A hard stop and redirect share one lock so /stop cannot race with an accepted correction and
- # accidentally turn itself into a retry. The blocking in-flight-commit wait runs BEFORE the atomic
- # claim edge (redirect lock still held); the destructive pending-commit cancel runs AFTER the claim
- # survives (#99758 P1).
- with _ic_lock(self, "_pending_redirect_lock"):
- _fence_cancel_before_commit(
- _fence(), when_in_flight=True, failure_log="Compression hard-cancel fence wait failed"
- )
- if require_generation is None:
- # No claim to race: publish WITHOUT the liveness lock (bare AIAgent stand-ins in other
- # suites lack the liveness seam and would AttributeError).
- _publish_interrupt_state()
- else:
- # Final mutation edge: claim consumption and the FIRST observable publication are ONE
- # activity-lock critical section, so either the claim survives and commits before any later
- # activity stamp, or the stamp landed first and the abort declines without publishing.
- with self._liveness_activity_lock():
- if getattr(self, "_turn_liveness_abort_claim", None) != require_generation:
- return False
- self._turn_liveness_abort_claim = None
- _publish_interrupt_state()
- _fence_cancel_before_commit(
- _fence(), when_in_flight=False, failure_log="Compression hard-cancel fence admission failed"
- )
- self._pending_redirect = None
- # Codex watches a private interrupt event rather than Hermes' per-thread flag.
- _request_interrupt = _ic_codex_method(self, "request_interrupt")
- if _request_interrupt is not None:
- try:
- _request_interrupt()
- except Exception:
- logger.debug("Failed to interrupt Codex app-server turn", exc_info=True)
- # Cron turns request on the conversation thread (no nested interrupt-worker deadlock); their client
- # is registered here so this cross-thread interrupt can still shut the sockets.
- _ic_abort_active_request(self, "interrupt_abort", "Failed to abort active inline request")
- # Scope the tool interrupt to this agent's execution thread so other in-process agents are unaffected.
- if self._execution_thread_id is not None:
- _set_interrupt(True, self._execution_thread_id, reason=tool_interrupt_reason)
- self._interrupt_thread_signal_pending = False
- else:
- # Interrupt arrived before run_conversation bound the execution thread: defer the tool-level
- # signal instead of targeting the caller thread.
- self._interrupt_thread_signal_pending = True
- _ic_signal_tool_workers(self, True, reason=tool_interrupt_reason)
- # Propagate interrupt to any running child agents (subagent delegation)
- with self._active_children_lock:
- children_copy = list(self._active_children)
- for child in children_copy:
- try:
- if hard_cancel:
- request_hard_interrupt(child, message, tool_reason=tool_interrupt_reason)
- else:
- child.interrupt(message)
- except Exception as e:
- logger.debug("Failed to propagate interrupt to child agent: %s", e)
- if not self.quiet_mode:
- print("\n⚡ Interrupt requested" + (f": '{message[:40]}...'" if message and len(message) > 40 else f": '{message}'" if message else ""))
- return True
- def hard_interrupt(self, message: Optional[str] = None, *, tool_reason: Optional[str] = None) -> None:
- """Explicit stop preserving the ``interrupt()`` ABI (frontends feature-detect this and fall back to
- legacy ``interrupt()`` for third-party agents). Bypasses dynamic dispatch: legacy subclasses may
- override interrupt(message=None) without hard_cancel."""
- InterruptControlMixin.interrupt(self, message, hard_cancel=True, tool_reason=tool_reason)
- def clear_interrupt(self, *, preserve_redirect: bool = False, hard_cancel: bool = False) -> bool:
- """Clear the interrupt request and per-thread tool signal. ``preserve_redirect`` is only for the
- conversation loop rebuilding the same logical turn after cancelling a model request.
- ``hard_cancel`` also drops an unconsumed pending steer: the aborted turn's next tool iteration
- will no longer happen, and re-injecting the note into the post-stop turn would surprise the
- user. Every other caller (redirect rebuild, error recovery, turn-boundary hygiene) continues
- this session, so the already-accepted steer must survive: it stays buffered for the existing
- drains — the pre-API inject, the post-batch append, or the finalizer's leftover handoff —
- instead of silently vanishing after the surface was told it was delivered."""
- with _ic_lock(self, "_pending_redirect_lock"):
- if preserve_redirect and not _ic_slot(self, "_pending_redirect_lock", "_pending_redirect"):
- return False
- self._interrupt_requested = False
- self._interrupt_message = self._tool_interrupt_reason = None
- getattr(self, "_hard_interrupt_requested", threading.Event()).clear()
- if not preserve_redirect:
- self._pending_redirect = None
- self._interrupt_thread_signal_pending = False
- if self._execution_thread_id is not None:
- _set_interrupt(False, self._execution_thread_id)
- _ic_signal_tool_workers(self, False)
- if hard_cancel:
- # Hard stop only (see docstring). The comment that used to run unconditionally here
- # claimed a hard interrupt supersedes the steer — but nothing gated this wipe on
- # hard_cancel, so a soft clear dropped a live user message with no trace.
- with _ic_lock(self, "_pending_steer_lock"):
- for ticket, _ in _steer_parts(self):
- if ticket is not None:
- _steer_fates(self)[ticket] = False
- _set_steer_parts(self, [])
- return True
- def steer(self, text: str, *, ticket=None) -> bool:
- """Queue user text for delivery as its own user row after the current tool batch finishes (no
- interrupt); multiple calls concatenate with newlines. Returns False for empty text. ``ticket``
- names the submission, so ``withdraw_steer`` can settle exactly it, whatever text others carry."""
- if not text or not text.strip():
- return False
- cleaned = text.strip()
- with _ic_lock(self, "_pending_steer_lock"):
- _set_steer_parts(self, _steer_parts(self) + [(ticket, cleaned)])
- # The turn no longer ran on its own (first-task-done detection reads this off the result).
- self._turn_user_intervened = True
- return True
- def redirect(self, text: str) -> bool:
- """Redirect the active turn without converting it into a new task: during a model request only that
- request is cancelled (completed messages kept, partial reasoning becomes assistant context, the
- correction is appended as a real user message, the loop retries); during tool execution it degrades
- to ``steer()``; Codex app-server uses native ``turn/steer``. False when no live turn / empty text."""
- if not text or not text.strip():
- return False
- cleaned = text.strip()
- _native_steer = _ic_codex_method(self, "request_steer")
- if _native_steer is not None:
- with _ic_lock(self, "_pending_redirect_lock"):
- if self._interrupt_requested:
- return False
- try:
- accepted = bool(_native_steer(cleaned))
- if accepted:
- self._turn_user_intervened = True
- return accepted
- except Exception:
- logger.debug("Codex app-server turn/steer failed", exc_info=True)
- return False
- # Never kill a tool to deliver guidance; the steer drain puts it on the final tool result.
- # A foreground terminal command would park that delivery until it exits (a 5-minute
- # `sleep` poller, a build), so ask the tool workers to YIELD: terminal hands the live
- # process to the background registry and returns; tools that don't yield are unaffected.
- if getattr(self, "_executing_tools", False):
- accepted = self.steer(cleaned)
- if accepted:
- tracker = getattr(self, "_tool_worker_threads", None)
- tracker_lock = getattr(self, "_tool_worker_threads_lock", None)
- if tracker is not None and tracker_lock is not None:
- with tracker_lock:
- worker_tids = list(tracker)
- for tid in worker_tids:
- _request_yield(tid)
- return accepted
- _model_active = getattr(self, "_model_request_active", None)
- with _ic_lock(self, "_pending_redirect_lock"):
- request_active = _model_active is not None and _model_active.is_set()
- if not request_active and not getattr(self, "_turn_redirect_open", False):
- return False
- existing = _ic_slot(self, "_pending_redirect_lock", "_pending_redirect")
- if self._interrupt_requested and not existing:
- return False
- self._pending_redirect = (
- f"{existing}\n\n[Additional user correction]\n{cleaned}" if existing else cleaned
- )
- self._turn_user_intervened = True
- if not request_active:
- # Preparation has no request to cancel. The iteration entry or
- # pre-call gate will append this correction before model use.
- return True
- self._interrupt_requested = True
- self._interrupt_message = None
- # Interrupt only the model request — no fan-out to tool workers / child agents as interrupt() does.
- _execution_thread_id = getattr(self, "_execution_thread_id", None)
- if _execution_thread_id is not None:
- _set_interrupt(True, _execution_thread_id)
- self._interrupt_thread_signal_pending = False
- else:
- self._interrupt_thread_signal_pending = True
- _ic_abort_active_request(self, "redirect_abort", "Failed to abort request for redirect")
- return True
- def _has_pending_redirect(self) -> bool:
- """Return whether an active-turn redirect is waiting to be applied."""
- with _ic_lock(self, "_pending_redirect_lock"):
- return bool(_ic_slot(self, "_pending_redirect_lock", "_pending_redirect"))
- def _drain_pending_redirect(self) -> Optional[str]:
- """Return and clear pending active-turn correction text."""
- with _ic_lock(self, "_pending_redirect_lock"):
- text = _ic_slot(self, "_pending_redirect_lock", "_pending_redirect")
- self._pending_redirect = None
- return text
- def _drain_pending_steer(self) -> Optional[str]:
- """Return the pending steer text (if any) and clear the slot; None when nothing is pending."""
- with _ic_lock(self, "_pending_steer_lock"):
- text = _ic_slot(self, "_pending_steer_lock", "_pending_steer")
- parts = _steer_parts(self)
- _set_steer_parts(self, [])
- for ticket, _ in parts:
- if ticket is not None:
- _steer_fates(self)[ticket] = True
- self._steer_last_drain = (text, parts) # requeue_drained_steer restores its tickets
- return text
- def _drain_leftover_steer(self) -> Optional[str]:
- """The turn finalizer's drain: return and clear the ticketless steers (they become the next user
- turn) and keep each ticketed one buffered. A transfer is never a follow-up a restart may discard:
- settlement withdraws it and its admission goes back to its own queue place."""
- with _ic_lock(self, "_pending_steer_lock"):
- parts = _steer_parts(self)
- _set_steer_parts(self, [part for part in parts if part[0] is not None])
- return "\n".join(text for ticket, text in parts if ticket is None) or None
- def withdraw_steer(self, ticket) -> bool:
- """Settle one ``steer(text, ticket=...)`` by identity (the gateway's transfer of a queued prompt):
- True when no drain handed it to the turn, either still buffered (taken out now; every other steer
- stays, whatever its text) or dropped unread; False once a drain consumed it, or for an unknown one."""
- if ticket is None:
- return False
- with _ic_lock(self, "_pending_steer_lock"):
- parts = _steer_parts(self)
- kept = [part for part in parts if part[0] != ticket]
- consumed = _steer_fates(self).pop(ticket, True)
- if len(kept) == len(parts):
- return consumed is False
- _set_steer_parts(self, kept)
- return True
|