run_turn_runner.py 92 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468
  1. """Per-turn callback runner (progress/status/voice/run_sync) for the gateway agent turn.
  2. ``TurnRunner`` owns the per-turn callbacks ``GatewayRunner._run_agent_inner`` binds. ``gateway.run``
  3. internals are imported lazily inside method bodies (import cycle), so ``patch("gateway.run.X")``
  4. keeps intercepting them at call time.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. import dataclasses
  9. import json
  10. import logging
  11. import queue
  12. import re
  13. import threading
  14. import time
  15. from contextlib import suppress
  16. from datetime import datetime
  17. from typing import TYPE_CHECKING, Any, Dict, List, Optional
  18. from agent.i18n import t
  19. from agent.interrupt_compat import _accepts_keyword
  20. from agent.replay_cleanup import canonicalize_replay_history
  21. from gateway.config import Platform
  22. from gateway.media_repair import repair_explicit_computer_use_media_paths
  23. from gateway.platforms.base import BasePlatformAdapter
  24. from gateway.platforms.base_exec_approval import ea_default_reason_text
  25. from gateway.turn_context import TurnContext
  26. from hermes_cli.config import cfg_get
  27. from hermes_cli.plugins_dispatch import stream_held
  28. from utils import is_truthy_value
  29. if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
  30. from gateway.run import GatewayRunner
  31. # Log-record parity with the origin module.
  32. logger = logging.getLogger("gateway.run")
  33. # Consecutive turns a session's persisted transcript may lag its live cached history before
  34. # _load_turn_history escalates the lag line from WARNING to ERROR (#114266: 11 days of WARNING).
  35. _TRANSCRIPT_LAG_ESCALATION_TURNS = 3
  36. # Exact refusals retained for older adapters/connectors without destination preflight.
  37. # Substring matching would also silence transient thread-resolution errors.
  38. _CARD_DESTINATION_REFUSALS = {
  39. "No Slack thread target",
  40. "slack task_card requires a thread anchor",
  41. "slack task_card requires a thread anchor (Slack streams are thread replies)",
  42. }
  43. def _renders_exec_approval_buttons(adapter_cls: type) -> bool:
  44. """True when the adapter class renders native approval buttons. BasePlatformAdapter subclasses
  45. say so through ``supports_exec_approval_buttons``; anything else (test doubles, relay-style
  46. duck types) counts when it defines ``send_exec_approval`` itself."""
  47. probe = getattr(adapter_cls, "supports_exec_approval_buttons", None)
  48. if callable(probe) and issubclass(adapter_cls, BasePlatformAdapter):
  49. return bool(probe())
  50. return getattr(adapter_cls, "send_exec_approval", None) is not None
  51. # Rendered on a native clarify card whose wait ended without a click (mirrors the notice the
  52. # Slack click handler shows on a dead entry).
  53. def _clarify_expired_notice() -> str:
  54. return t("gateway.clarify.expired")
  55. # run_conversation result keys the gateway projection carries verbatim so a finite viewer's
  56. # `-z --usage-file` ledger (hermes_cli/oneshot._USAGE_KEYS) reads what the in-process path did.
  57. _LEDGER_PASSTHROUGH_KEYS = (
  58. "estimated_cost_usd", "cost_status", "cost_source", "cache_read_tokens", "cache_write_tokens",
  59. "reasoning_tokens", "total_tokens", "provider", "turn_exit_reason", "service_tier",
  60. )
  61. # Beside ``failure_reason`` and ``error``: what ``agent.error_surface.build_error_surface_from_result`` reads,
  62. # so the authority's ``message.complete`` carries the same error card the TUI path builds;
  63. # ``usage_window_message`` also marks a window-plan refusal for the chat rewording.
  64. _FAILURE_DESCRIPTOR_KEYS = ("free_tier", "failure_retryable", "failure_resets_at", "usage_window_message")
  65. def _finite_viewer_turn() -> bool:
  66. from gateway.session_finite import finite_turn_required
  67. return finite_turn_required() is True
  68. def _persisted_turn(agent, agent_history, result, compressions) -> dict | None:
  69. """``message.complete.persisted_turn`` for this turn, the receipt the TUI gateway's own turns
  70. publish: clients bind the streamed reply to its stored row instead of comparing words. The
  71. agent copies ``conversation_history`` into its message list, so an untouched pre-turn prefix
  72. is the same dicts (transcript rows here carry no ``_row_id`` to compare)."""
  73. from agent.persisted_turn_receipt import persisted_turn_receipt
  74. if not isinstance(result, dict):
  75. return None
  76. status = ("interrupted" if result.get("interrupted")
  77. else "error" if result.get("failed") or result.get("error") else "complete")
  78. return persisted_turn_receipt(
  79. result.get("messages"), getattr(agent, "_persist_user_message_idx", None), agent_history,
  80. result.get("final_response"), status,
  81. compression_unchanged=type(compressions) is int and compressions == getattr(
  82. getattr(agent, "context_compressor", None), "compression_count", None),
  83. prefix_row_matches=lambda before, after: before is after)
  84. class _NoStreamConsumer(RuntimeError):
  85. """Raised by the delta callback when nothing consumed the text; ``_call_quietly`` turns it into
  86. 'not delivered' so the agent's partial-delivery accounting matches what the user saw."""
  87. class _ExecApprovalDeclined(RuntimeError):
  88. """The connector refused the approval card's destination.
  89. Raised (not returned) so it propagates out of `_approval_notify_sync` to
  90. `_await_gateway_decision`, whose notify-failure path drops the central
  91. approval queue entry and unblocks the waiting tool. A plain return
  92. suppressed the text fallback but left that entry pending.
  93. """
  94. from gateway.run_turn_progress import GatewayTurnProgressMixin
  95. from gateway.session_execution import GatewaySessionAgentMixin
  96. def _settle_api_runtime_lock(agent, result, captured):
  97. """A confirmed API model lock settles only on the runtime it named (main's
  98. ``_finish_turn_result``): the served runtime rides the usage, a mismatch fails the turn."""
  99. from gateway.session_api_turn import api_runtime_locked
  100. api = api_runtime_locked()
  101. if api is None or not isinstance(result, dict):
  102. return
  103. settings = api['settings']
  104. try:
  105. captured['usage']['runtime'] = api['adapter']._turn_runtime_metadata(
  106. agent, route=settings.get('route'), requested_runtime=settings.get('requested_runtime'),
  107. route_source=settings.get('route_source') or '', confirmed_runtime_lock=True)
  108. except RuntimeError as exc:
  109. result.update(final_response='', failed=True, completed=False, error=str(exc))
  110. def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str, str, bool]:
  111. """``(chat text, failure_reason, retryable)`` for a model/credential resolution failure before the turn."""
  112. from agent.error_classifier import FailoverReason
  113. from hermes_cli.auth import AuthError, is_rate_limited_auth_error
  114. from hermes_cli.runtime_provider_custom import LocalRuntimeUnavailableError
  115. from hermes_state_runtime import RuntimeStoreError
  116. auth_permanent = FailoverReason.auth_permanent.value
  117. if isinstance(exc, LocalRuntimeUnavailableError):
  118. # The chosen model is configured; its local server is down. Not a credential verdict, so
  119. # the client must not offer "Update API key" or sign-in: the sentence names the fix.
  120. return f"⚠️ {exc}", exc.error_code, exc.retryable
  121. if isinstance(exc, RuntimeStoreError) and exc.reason == 'launch_credentials_unavailable':
  122. # The session's launch key is gone (a restart revoked a CLI --api-key, a route key changed):
  123. # the shared refusal copy names /new, never a resend.
  124. from agent.turn_failure_copy import launch_credentials_copy
  125. return launch_credentials_copy(exc), 'launch_credentials_unavailable', False
  126. if isinstance(exc, RuntimeStoreError):
  127. # Session-policy refusals carry a stable reason code (e.g. a CLI launch key revoked by
  128. # daemon restart): keep it in the reply so clients can act on it, and do not suggest
  129. # /login — the profile's own credentials were never in play.
  130. return (f"⚠️ This session's launch credentials are no longer available ({exc.reason}), so this "
  131. "message wasn't processed. Start a new session from the CLI to bind them again.", auth_permanent, False)
  132. if is_rate_limited_auth_error(exc.__cause__):
  133. # Quota cap with valid credentials: /login cannot help; name the reset window (#89401).
  134. from gateway.run import _gateway_provider_error_reply
  135. return _gateway_provider_error_reply(str(exc)), auth_permanent, False
  136. if platform == Platform.LOCAL:
  137. # The local operator reads the terminal: the resolver's own sentence ("provider 'custom'
  138. # resolved without credentials ...") is the diagnosis; /login is not.
  139. return f"⚠️ {exc}", auth_permanent, False
  140. seen: set[int] = set()
  141. host = auth = False
  142. link: BaseException | None = exc
  143. while link is not None and id(link) not in seen:
  144. seen.add(id(link))
  145. host = host or isinstance(link, OSError)
  146. auth = auth or isinstance(link, AuthError)
  147. link = link.__cause__ or link.__context__
  148. if host and not auth:
  149. # A host-side failure (a CA bundle deleted under a long-lived process, an unreadable file)
  150. # with no credential verdict: /login cannot fix it, the gateway log can (#140344).
  151. # host_error sits on the gateway layer, so clients don't offer "Switch provider"; not retryable,
  152. # since a deleted bundle or unreadable store fails the same way until the host is fixed.
  153. return t("gateway.errors.generic_failed"), "host_error", False
  154. return t("gateway.errors.no_credentials"), auth_permanent, False
  155. class TurnRunner(GatewayTurnProgressMixin, GatewaySessionAgentMixin):
  156. """Per-turn collaborator carrying ``GatewayRunner._run_agent_inner``'s tool-progress callbacks."""
  157. # ``None`` is a legitimate state: a turn with no owning session authority (messaging
  158. # adapters without a canonical session) publishes no execution events and takes no
  159. # controls snapshot. Class-level so every publish/snapshot seam can read it unconditionally.
  160. _approval_owner = None
  161. def __init__(self, runner: GatewayRunner, ctx: TurnContext) -> None:
  162. self._runner = runner
  163. self._ctx = ctx
  164. from gateway.session_authorities import active_authority
  165. authority = active_authority(runner)
  166. if authority is not None:
  167. source = getattr(ctx, 'source', None)
  168. owner_id = (source.chat_id if getattr(source, 'platform', None) == Platform.LOCAL
  169. else authority.logical_owner(ctx.session_id))
  170. if owner_id in authority.sessions:
  171. generation = authority.db.get_session(owner_id)["runtime_generation"]
  172. self._approval_owner = (authority, owner_id, generation)
  173. def _publish_execution(self, event_type, payload):
  174. if self._approval_owner is not None:
  175. authority, session_id, generation = self._approval_owner
  176. return authority.publish_execution(
  177. session_id, generation, event_type, self._stamp_segment(generation, event_type, payload))
  178. return False
  179. def _stamp_segment(self, generation, event_type, payload):
  180. """The turn's wire identity on its reply frames (tui_gateway/stream_segments.py), so a viewer
  181. upserts a re-sent delta or seal instead of painting a second copy (#144609). The same
  182. stamping rule as tui_gateway's ``_emit``, with one cursor per execution generation."""
  183. from tui_gateway.stream_segments import TurnSegments, stamp_reply_frame
  184. segments = getattr(self, "_segments", None)
  185. if segments is None or segments[0] != generation:
  186. segments = self._segments = (generation, TurnSegments())
  187. return stamp_reply_frame(segments[1], event_type, payload)
  188. def _publish_api_tool(self, event_type, call_id, tool_name, args, result=None):
  189. """The retained tool payload reaches the API observers of this exact admission only;
  190. the shared viewer stream keeps its ID-correlated frames."""
  191. if self._approval_owner is not None:
  192. from gateway.session_api_turn import publish_api_tool_event
  193. authority, session_id, generation = self._approval_owner
  194. publish_api_tool_event(authority, session_id, generation, event_type,
  195. str(call_id or ""), str(tool_name or "tool"), args, result)
  196. # ── stream consumer / interim commentary wiring ─────────────────────────────────────────
  197. def _setup_stream_consumer(self, platform_key):
  198. ctx = self._ctx
  199. if ctx.mute_notification_reply:
  200. return None, None, None, False
  201. stream_consumer = None
  202. # The streaming-TTS consumer is created on the outer loop thread before run_sync launches;
  203. # run_sync only reads it via the holder for delta-callback wiring.
  204. stts = ctx.streaming_tts_consumer_holder[0]
  205. scfg = getattr(getattr(self._runner, 'config', None), 'streaming', None)
  206. if scfg is None:
  207. from gateway.config import StreamingConfig
  208. scfg = StreamingConfig()
  209. # display.platforms.<plat>.streaming may disable streaming per platform; None = follow global.
  210. plat_streaming = ctx.resolve_display_setting(ctx.user_config, platform_key, "streaming")
  211. # A loaded ``hold_stream`` transform rewrites this session's reply before the user may see it:
  212. # no drafts, edits, native stream or commentary bubbles; the rewrite goes out as one plain
  213. # final while the typing indicator runs (the agent also withholds every delta).
  214. held = stream_held()
  215. want_stream_deltas = not held and not ctx.scheduled_heartbeat and scfg.enabled_for(plat_streaming)
  216. # A finite (`-z`, `chat -q`) viewer prints only the terminal reply, like the classic quiet
  217. # CLI that never wired commentary. Commentary it cannot see must not be recorded as a
  218. # delivery, or a final that arrives as commentary (codex app-server) suppresses the reply.
  219. want_interim_messages = (bool(ctx.interim_assistant_messages_enabled) and not ctx.scheduled_heartbeat
  220. and not held and not _finite_viewer_turn())
  221. if want_stream_deltas or want_interim_messages:
  222. try:
  223. from gateway.stream_consumer import GatewayStreamConsumer
  224. adapter = self._runner._delivery_adapter_for(ctx.source)
  225. if adapter:
  226. supports_incremental_stream = (
  227. getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True)
  228. or bool(getattr(adapter, "SUPPORTS_NATIVE_STREAMING", False))
  229. )
  230. consumer_stream_deltas = want_stream_deltas and supports_incremental_stream
  231. consumer_cfg, pause_typing_before_finalize = self._runner._build_stream_consumer_config(
  232. ctx.source, scfg, adapter,
  233. # A complete commentary message needs no edit cursor. Keeping it on the
  234. # consumer records what reached non-editable platforms, so an interim
  235. # callback carrying the final answer participates in final-send dedup.
  236. on_missing_cursor="fallback" if want_interim_messages else "raise",
  237. )
  238. stream_consumer = GatewayStreamConsumer(
  239. adapter=adapter, chat_id=ctx.source.chat_id, config=consumer_cfg,
  240. metadata=ctx._status_thread_metadata,
  241. on_new_message=(
  242. (lambda: ctx.progress_queue.put(("__reset__",))) if ctx.progress_queue is not None else None
  243. ),
  244. on_before_finalize=pause_typing_before_finalize,
  245. initial_reply_to_id=ctx.event_message_id, run_still_current=ctx._run_still_current,
  246. )
  247. ctx.stream_consumer_holder[0] = stream_consumer
  248. # #105341: a consumer created only for interim commentary (text streaming off)
  249. # is never fed the final reply's deltas — mark it so the duplicate-risk
  250. # diagnostic in ``_run_agent_mark_streamed_delivery`` stays silent.
  251. stream_consumer.stream_deltas_enabled = consumer_stream_deltas
  252. except Exception as err:
  253. logger.debug("Could not set up stream consumer: %s", err)
  254. # Deltas tee to the stream consumer (when text streaming is on) and to streaming TTS.
  255. delta_sinks = [
  256. sc for sc in (
  257. stream_consumer if stream_consumer and stream_consumer.stream_deltas_enabled else None,
  258. stts,
  259. ) if sc is not None
  260. ]
  261. stream_delta_cb = None
  262. local_viewer_route = getattr(ctx.source, "platform", None) == Platform.LOCAL
  263. if delta_sinks or self._approval_owner is not None:
  264. def stream_delta_cb(text: Optional[str]) -> None:
  265. if not ctx._run_still_current():
  266. return
  267. delivered = bool(delta_sinks)
  268. if text is not None:
  269. delivered = self._publish_execution("message.delta", {"text": text}) or delivered
  270. elif local_viewer_route:
  271. # The segment boundary (main's ``_stream_delta(None)`` closes the reply box
  272. # before tool lines); messaging routes only close their native stream.
  273. self._publish_execution("message.delta", {"text": None})
  274. for sink in delta_sinks:
  275. sink.on_delta(text)
  276. if text is not None and (not delivered or _finite_viewer_turn()):
  277. # Nobody the user is looking at saw this text: no live viewer/observer/adapter
  278. # sink, or a finite (`-z`, `chat -q`) viewer, which prints the terminal reply
  279. # only. The agent must treat it as undelivered, else a stream that dies here is
  280. # stitched as a "partial delivery" nobody received (classic -q parity).
  281. raise _NoStreamConsumer()
  282. def interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None:
  283. if not ctx._run_still_current():
  284. return
  285. unseen = False
  286. if local_viewer_route and not already_streamed and str(text or "").strip():
  287. # LocalSessionAdapter.send publishes nothing, so unstreamed commentary handed to the
  288. # consumer would be recorded as delivered while no viewer saw it. A codex app-server
  289. # final that arrives with no deltas then suppresses the reply (message.complete "").
  290. # Publish the in-process TUI's message.interim contract instead, and count the text
  291. # as delivered only when a viewer or observer actually took it.
  292. payload = {"text": text, "already_streamed": False}
  293. unseen = not self._publish_execution("message.interim", payload)
  294. if stts is not None:
  295. # Flush accepted deltas; completed commentary is a separate speech segment.
  296. stts.on_delta(None)
  297. if not already_streamed:
  298. stts.on_delta(text)
  299. stts.on_delta(None)
  300. if unseen:
  301. return
  302. if stream_consumer is not None:
  303. stream_consumer.on_segment_break() if already_streamed else stream_consumer.on_commentary(text)
  304. elif not already_streamed and ctx._status_adapter and str(text or "").strip():
  305. self._send_status_text(text, ctx._status_thread_metadata, "interim_assistant_callback scheduling error")
  306. return stream_consumer, stream_delta_cb, interim_assistant_cb, want_interim_messages
  307. # ── per-turn agent wiring ───────────────────────────────────────────────────────────────
  308. def _notice_callback_sync(self, notice) -> None:
  309. """Credits / out-of-band notices (usage bands, depletion, restored) fire from the agent's
  310. sync worker thread; hop onto the gateway loop. Fired-once latch lives on the cached agent."""
  311. from gateway.run import render_notice_line
  312. from gateway.warning_notifications import is_diagnostic_notice, render_notification
  313. from gateway.bot_peer_notices import automatic_notices_allowed
  314. if self._ctx.mute_notification_reply or not self._status_live() or not automatic_notices_allowed(self._ctx.source):
  315. return
  316. # A usage-window limit line repeats the turn's own reply, which carries the sentence and the
  317. # switch button (a chat message cannot be cleared, so it would only pile up). The 90% heads-up
  318. # is a desktop / terminal nudge; in a chat it is noise (/usage has the numbers).
  319. key = str(getattr(notice, "key", "") or "")
  320. if key == "credits.window_limit" or key.startswith("credits.window."):
  321. return
  322. diagnostic = is_diagnostic_notice(notice)
  323. def present():
  324. try:
  325. line = render_notice_line(notice)
  326. if key.startswith("credits."):
  327. # Window plans (e.g. a paused plan's line): a chat's /topup has no credits settings.
  328. from agent.plan_limits import portal_wording
  329. line = portal_wording(line)
  330. except Exception:
  331. logger.debug("render_notice_line failed", exc_info=True)
  332. return
  333. if line:
  334. self._schedule(self._runner._deliver_platform_notice(self._ctx.source, line), "notice_callback delivery scheduling error")
  335. render_notification(present, platform=self._ctx.source.platform,
  336. user_config=self._ctx.user_config, diagnostic=diagnostic)
  337. def _make_bg_review_callbacks(self):
  338. """(send, release): background-review messages ("💾 Memory updated") are held until the
  339. adapter's post-delivery hook releases them after the main response lands."""
  340. from gateway.run import _interim_metadata, _non_conversational_metadata
  341. ctx = self._ctx
  342. release_evt = threading.Event()
  343. pending: list[str] = []
  344. pending_lock = threading.Lock()
  345. owner = self._approval_owner
  346. local_viewer = owner is not None and getattr(ctx.source, "platform", None) == Platform.LOCAL
  347. def deliver(message: str) -> None:
  348. if local_viewer and owner is not None:
  349. # LocalSessionAdapter.send publishes nothing, so a local viewer only sees the
  350. # summary as the in-process TUI's review.summary frame. Not generation-fenced:
  351. # the review fork finishes after this turn's running claim has settled.
  352. authority, session_id, _generation = owner
  353. live = authority.sessions.get(session_id)
  354. if live is not None:
  355. live.event_stream.publish(session_id, {"text": str(message)}, event_type="review.summary")
  356. return
  357. if self._status_live():
  358. self._send_status_text(
  359. message,
  360. _interim_metadata(_non_conversational_metadata(ctx._status_thread_metadata, platform=ctx.source.platform)),
  361. "background_review_callback scheduling error",
  362. )
  363. def release() -> None:
  364. release_evt.set()
  365. with pending_lock:
  366. queued = list(pending)
  367. pending.clear()
  368. for message in queued:
  369. deliver(message)
  370. def send(message: str) -> None:
  371. if local_viewer and owner is not None:
  372. # The review fork reports from its own thread while the drain may still be
  373. # settling this turn. Hold the row until the authority published the turn's
  374. # message.complete (it clears the execution stamp only after that), the order
  375. # the in-process TUI produced. Never fenced: a late summary still reaches viewers.
  376. authority, session_id, generation = owner
  377. live = authority.sessions.get(session_id)
  378. deadline = time.monotonic() + 120
  379. while (live is not None and time.monotonic() < deadline
  380. and live.event_stream.execution.get("execution_generation") == generation):
  381. time.sleep(0.05)
  382. deliver(message)
  383. return
  384. if not self._status_live():
  385. return
  386. if not release_evt.is_set():
  387. with pending_lock:
  388. if not release_evt.is_set():
  389. pending.append(message)
  390. return
  391. deliver(message)
  392. return send, release
  393. @staticmethod
  394. def _merge_turn_request_overrides(agent, turn_route) -> None:
  395. """Merge, never overwrite: init-time request overrides (e.g. a custom provider's extra_body)
  396. must survive every reused-agent turn. Drop only the PREVIOUS turn's routing overrides before
  397. layering this turn's, so stale per-turn values never linger."""
  398. overrides = dict(getattr(agent, "request_overrides", {}) or {})
  399. for key, value in (getattr(agent, "_gateway_turn_request_overrides", {}) or {}).items():
  400. if overrides.get(key) == value:
  401. overrides.pop(key, None)
  402. turn_overrides = dict(turn_route.get("request_overrides") or {})
  403. overrides.update(turn_overrides)
  404. agent.request_overrides = overrides
  405. agent._gateway_turn_request_overrides = turn_overrides
  406. def _wire_turn_agent_callbacks(self, agent, turn_route, reasoning_config,
  407. stream_delta_cb, interim_assistant_cb, want_interim_messages):
  408. """Per-message state (callbacks, reasoning, voice route) — never baked into the cached agent."""
  409. ctx = self._ctx
  410. runner = self._runner
  411. agent._notification_config = ctx.user_config
  412. agent._notification_platform = ctx.source.platform
  413. # ALWAYS attached (never gated to None): its body gates each event class, and subagent-
  414. # failure notices must fire even with tool_progress/thinking off.
  415. agent.tool_progress_callback = ctx.progress_callback
  416. # Discord's one-time voice ack and Slack's task cards both ride the authoritative start
  417. # callback, so neither infers identity from tool names.
  418. agent.tool_start_callback = (
  419. (ctx.native_tool_start_callback or ctx.voice_ack_callback)
  420. if (ctx._voice_ack_guild[0] is not None or ctx._native_slack_task_cards) else None
  421. )
  422. agent.tool_complete_callback = ctx.native_tool_complete_callback if ctx._native_slack_task_cards else None
  423. if self._approval_owner is not None:
  424. agent.tool_start_callback = self.combined_tool_start_callback
  425. agent.tool_complete_callback = self.combined_tool_complete_callback
  426. agent.step_callback = ctx._step_callback_sync if (ctx._hooks_ref.loaded_hooks or self._approval_owner is not None) else None
  427. agent.stream_delta_callback = stream_delta_cb
  428. agent.interim_assistant_callback = interim_assistant_cb if want_interim_messages else None
  429. from gateway.run_turn_progress import local_viewer_turn
  430. agent.status_callback, agent.notice_callback = ctx._status_callback_sync, (
  431. self._local_notice_callback_sync if local_viewer_turn(self) else self._notice_callback_sync)
  432. agent.reasoning_callback = None # per turn: a reused agent never keeps an earlier API turn's sink
  433. if self._approval_owner is not None:
  434. from gateway.session_api_turn import api_execution, wire_api_observers
  435. if api_execution.get() is not None:
  436. # The API request's SSE observes reasoning/status/commentary through its admission.
  437. wire_api_observers(agent, self._approval_owner, want_interim_messages)
  438. # A platform send can't be retracted; a local viewer withdraws the keyed notice.
  439. agent.notice_clear_callback = self._local_notice_clear_callback_sync if local_viewer_turn(self) else None
  440. agent.event_callback = ctx._event_callback_sync
  441. agent.reasoning_config, agent.service_tier = reasoning_config, runner._service_tier
  442. _load_daybreak = getattr(runner, "_load_daybreak", None) # absent on minimal test runners
  443. agent.daybreak = callable(_load_daybreak) and _load_daybreak(ctx.source) is True
  444. self._merge_turn_request_overrides(agent, turn_route)
  445. from gateway.session_surface import arm_surface_turn
  446. # auxiliary.voice_chat route (a voice note, or an admission committed as a voice turn) and
  447. # this turn's must-deliver notes; a managed worker arms its agent through the same helper.
  448. arm_surface_turn(agent, runner._consume_pending_turn_sidecar_notes(ctx.session_key), voice_turn=ctx.voice_turn)
  449. agent.background_review_callback, bg_release = self._make_bg_review_callbacks()
  450. # Register the release hook on the adapter so base.py's finally block fires it after the
  451. # main response is delivered.
  452. if ctx._status_adapter and ctx.session_key:
  453. if getattr(type(ctx._status_adapter), "register_post_delivery_callback", None) is not None:
  454. ctx._status_adapter.register_post_delivery_callback(ctx.session_key, bg_release, generation=ctx.run_generation)
  455. else:
  456. pdc = getattr(ctx._status_adapter, "_post_delivery_callbacks", None)
  457. if pdc is not None:
  458. pdc[ctx.session_key] = bg_release
  459. # display.memory_notifications: off | on (generic "💾 Memory updated", default) | verbose.
  460. # `display:` present-but-null yields None, not the {} default (same `or {}` guard as
  461. # display_config.py / runtime_footer.py).
  462. mem_notif = (ctx.user_config.get("display") or {}).get("memory_notifications")
  463. if isinstance(mem_notif, bool):
  464. mem_notif = "on" if mem_notif else "off"
  465. agent.memory_notifications = str(mem_notif).lower() if mem_notif else "on"
  466. agent.clarify_callback = self._clarify_callback_sync
  467. # Thinking between tool calls is independent of tool_progress mode (Mattermost opts in
  468. # per platform so global scratch-text doesn't leak into threads).
  469. agent.thinking_progress = ctx._thinking_enabled
  470. if ctx.mute_notification_reply:
  471. # Controls and operational event/step callbacks remain wired. These
  472. # presentation callbacks are rebound on every next turn.
  473. agent.tool_progress_callback = None
  474. agent.tool_start_callback = None
  475. agent.tool_complete_callback = None
  476. # Keep diagnostic observers installed; concrete sinks veto display.
  477. agent.stream_delta_callback = None
  478. agent.interim_assistant_callback = None
  479. agent.thinking_progress = False
  480. ctx.agent_holder[0] = agent # interrupt support
  481. if self._approval_owner is not None:
  482. authority, owner_id, generation = self._approval_owner
  483. authority.adopt_agent(owner_id, generation, agent)
  484. # The titler fires from the turn prologue, so attach the rename lane before the run.
  485. self._attach_session_title_callback(agent, ctx)
  486. # Publish turn ownership for /stop, /new, disconnect and shutdown interrupts; older session
  487. # processes are outside this baseline and remain alive.
  488. agent._gateway_turn_process_task_id, agent._gateway_turn_process_baseline = ctx.process_task_id, ctx.process_baseline
  489. ctx.tools_holder[0] = getattr(agent, "tools", None) # transcript logging
  490. # ── blocking prompts from the agent thread (approval / clarify) ─────────────────────────
  491. def _close_native_stream_boundary(self, reason: str, placeholder: str | None = None, reopen: bool = False) -> bool:
  492. """Native-streaming platforms (e.g. WeCom): an interrupting interaction (approval or clarify
  493. prompt) must finalize the current stream first, or post-interaction output keeps updating the
  494. OLD bubble above the prompt. Runs on the agent thread; the consumer serializes via its queue."""
  495. sc = self._stream_consumer()
  496. if not (sc and getattr(sc, "_use_native_streaming", False)):
  497. return True
  498. cancelled_flag = None
  499. try:
  500. boundary = sc.close_for_approval_prompt(placeholder, reason=reason, reopen=reopen)
  501. # Returns (future, cancelled_flag) or just a future.
  502. if isinstance(boundary, tuple):
  503. boundary, cancelled_flag = boundary
  504. if not hasattr(boundary, "result"):
  505. return True
  506. ok = boundary.result(timeout=10)
  507. if not ok:
  508. logger.warning(
  509. "%s boundary failed to close stream properly — "
  510. "prompt may still appear in typing bubble", reason,
  511. )
  512. return bool(ok)
  513. except (TimeoutError, Exception) as err:
  514. if cancelled_flag is not None:
  515. cancelled_flag["cancelled"] = True
  516. logger.warning("%s boundary timed out or failed: %s", reason, err)
  517. return False
  518. async def _send_shared_clarify(self, entry, **kwargs):
  519. result = await self._ctx._status_adapter.send_clarify(**kwargs)
  520. if self._approval_owner is not None and result.success:
  521. authority, session_id, generation = self._approval_owner
  522. from gateway import local_viewer_events
  523. authority.register_clarify(session_id, generation, entry,
  524. timeout_s=local_viewer_events.prompt_timeout_s(self, "clarify"))
  525. return result
  526. def _clarify_callback_sync(self, questions) -> dict:
  527. """Answer the clarify tool's questions (clarify_tool's synchronous contract): one card per
  528. question, stop at the first the user never answers. The stream/typing re-arm waits for the
  529. last question — between two cards it only opens a bubble the next boundary closes."""
  530. from gateway.run_turn_runner_clarify_delivery import UNDELIVERED, UNDELIVERED_DECLINED, UNDELIVERED_NO_SURFACE
  531. from tools.clarify_gateway import CANCELLED, SKIPPED
  532. if _finite_viewer_turn():
  533. # `chat -q` / `-z`: the viewer detaches on the first prompt, so nobody can answer.
  534. # Answer headless at once, as main's _single_query_clarify_callback did in process.
  535. from gateway.session_finite import finite_clarify_reply
  536. return finite_clarify_reply()
  537. from gateway.session_cli_turn import clarify_batch_turn
  538. if clarify_batch_turn(self):
  539. # Main's batch panel (classic CLI, Ink TUI) shows every question at once: one request.
  540. from gateway.session_clarify_batch import ask_clarify_batch
  541. return ask_clarify_batch(self, questions)
  542. answers: dict[str, Any] = {}
  543. reply: dict[str, Any] = {"answers": answers, "outcome": "submitted"}
  544. last = len(questions) - 1
  545. # The typed-"skip" hint is for messaging adapters, where a chat reply answers the card. A local
  546. # viewer (TUI / Desktop / CLI) renders main's modal with its own skip control, unhinted.
  547. platform = getattr(getattr(self._ctx, "source", None), "platform", None)
  548. hint = "" if platform == Platform.LOCAL else f"\n{t('gateway.clarify.skip_hint')}"
  549. for index, entry in enumerate(questions):
  550. question = f"{entry['question']}{hint}"
  551. raw, answered = self._ask_clarify_question(
  552. question, entry["choices"], bool(entry["multi_select"]), rearm=index == last)
  553. if raw == CANCELLED:
  554. reply["outcome"] = "cancelled"
  555. break
  556. if not answered:
  557. # The surface's own no-answer text ("could not be delivered", "did not respond
  558. # within Nm") rides along as ``notice``: blank answers alone read as user
  559. # inactivity, which is the misreport #112684 describes for an undelivered card.
  560. undelivered = raw in (UNDELIVERED, UNDELIVERED_DECLINED, UNDELIVERED_NO_SURFACE)
  561. reply.update(outcome="undelivered" if undelivered else "timed_out", notice=raw)
  562. break
  563. answers[entry["qid"]] = None if raw == SKIPPED else raw
  564. return reply
  565. def _ask_clarify_question(self, question, choices, multi_select, rearm: bool = True) -> tuple[str, bool]:
  566. """One card: register, send, wait, then retire it (no answer) or re-arm (answer).
  567. Returns ``(response, answered)``; the caller decides what "no answer" means."""
  568. from gateway.run_turn_runner_clarify_delivery import (
  569. UNDELIVERED_NO_SURFACE, _clarify_send_then_wait, text_fallback_coro)
  570. from tools import clarify_gateway as clarify_mod
  571. import uuid
  572. ctx = self._ctx
  573. if not ctx._status_adapter:
  574. # Nothing can render the question: say so, or the batch's blank answers read as
  575. # user inactivity (#112684).
  576. return UNDELIVERED_NO_SURFACE, False
  577. session_key = ctx.session_key or ""
  578. clarify_id = uuid.uuid4().hex[:10]
  579. choices = list(choices) if choices else None
  580. send_kwargs = dict(
  581. chat_id=ctx._status_chat_id, question=question, choices=choices, clarify_id=clarify_id,
  582. session_key=session_key, metadata=ctx._status_thread_metadata,
  583. )
  584. def _text_fallback():
  585. """Schedule the plain-text prompt when the native card cannot render; None = no such path."""
  586. coro = text_fallback_coro(ctx._status_adapter, **send_kwargs)
  587. return None if coro is None else self._schedule(coro, "Clarify text fallback failed to schedule")
  588. entry = clarify_mod.register(
  589. clarify_id=clarify_id, session_key=session_key, question=question, choices=choices,
  590. multi_select=bool(multi_select),
  591. )
  592. # Unlike approval, clarify passes reopen=True so the continuation re-opens a native stream
  593. # below the question; if the re-seed fails the consumer degrades to send() automatically.
  594. self._close_native_stream_boundary("Clarify", t("gateway.clarify.native_stream_placeholder"), reopen=True)
  595. # Pause typing: a "thinking..." status must not obscure the prompt or block an "Other" reply
  596. # on platforms that disable input while typing (Slack Assistant).
  597. with suppress(Exception):
  598. ctx._status_adapter.pause_typing_for_chat(ctx._status_chat_id)
  599. # Ordering barrier: flush buffered assistant prose BEFORE the poll, which goes out on a
  600. # separate agent-thread-blocking path and would otherwise render ABOVE its own explanation.
  601. # Best-effort + short timeout so the agent thread never hangs if the consumer isn't running.
  602. flush = getattr(self._stream_consumer(), "flush_pending_sync", None)
  603. try:
  604. if callable(flush):
  605. flush(timeout=3.0)
  606. except Exception:
  607. logger.debug("Stream-consumer flush before clarify prompt failed", exc_info=True)
  608. fut = self._schedule(
  609. self._send_shared_clarify(entry, **send_kwargs),
  610. "Clarify send failed to schedule",
  611. )
  612. # Boundary rule (see _approval_send_outcome): a send timeout is AMBIGUOUS — the card may
  613. # have posted with a late ack. Only a definitive failure tears down the registration;
  614. # ambiguous falls through to the bounded wait so a late reply resolves. A definitive
  615. # failure — immediate or late — retries once as plain text before giving up.
  616. response, answered = _clarify_send_then_wait(
  617. fut, clarify_id=clarify_id, session_key=session_key, clarify_mod=clarify_mod,
  618. fallback=_text_fallback)
  619. if self._approval_owner is not None:
  620. authority, session_id, generation = self._approval_owner
  621. authority.sessions[session_id].controls.snapshot(session_id, generation)
  622. # Branch on the explicit flag, never on the text: a real answer can start with '[' (a
  623. # "[A] staging" label, "[urgent] ..." free text) and must not be mistaken for a sentinel.
  624. if not answered:
  625. # No answer arrived (timeout, /new, run end): retire the native card so it stops
  626. # looking answerable. Adapters without a persistent card have no such method.
  627. retire = getattr(type(ctx._status_adapter), "retire_clarify_card", None)
  628. if callable(retire):
  629. self._schedule(
  630. retire(ctx._status_adapter, clarify_id, _clarify_expired_notice()),
  631. "Clarify card retire failed to schedule")
  632. elif rearm:
  633. # Reopen typing IMMEDIATELY, not on the LLM's first post-answer token (native streaming
  634. # otherwise re-seeds lazily on the first delta: ~48s of dead air). request_reopen_seed is
  635. # a no-op outside the reopen-pending native state.
  636. sc = self._stream_consumer()
  637. if sc is not None:
  638. try:
  639. sc.request_reopen_seed()
  640. except Exception:
  641. logger.debug("request_reopen_seed after clarify answer failed", exc_info=True)
  642. try:
  643. ctx._status_adapter.resume_typing_for_chat(ctx._status_chat_id)
  644. except Exception:
  645. logger.debug("resume_typing_for_chat after clarify answer failed", exc_info=True)
  646. return response, answered
  647. def _approval_notify_sync(self, approval_data: dict) -> None:
  648. if self._approval_owner is not None:
  649. authority, session_id, generation = self._approval_owner
  650. authority.check_approval_generation(session_id, generation)
  651. # A native decline must finish before any observer can authorize work.
  652. self._render_approval_sync(approval_data)
  653. if self._approval_owner is not None:
  654. from gateway import local_viewer_events
  655. authority.register_approval(session_id, generation, self._ctx.session_key, approval_data,
  656. timeout_s=local_viewer_events.prompt_timeout_s(self, "approval"))
  657. def _render_approval_sync(self, approval_data: dict) -> None:
  658. """Send the approval request from the agent thread: the adapter's interactive button
  659. approvals (``send_exec_approval``) when available, else plain text with ``/approve`` steps."""
  660. from gateway.run import _approval_send_outcome, _format_exec_approval_fallback, _interim_metadata, _redact_approval_command
  661. from gateway.run_turn_runner_approval_settle import register_timeout_notice
  662. ctx = self._ctx
  663. adapter = ctx._status_adapter
  664. # Slack's assistant_threads_setStatus disables the compose box, so the user can't type
  665. # /approve while "is thinking..." shows. Pausing stops _keep_typing re-setting it; resumed
  666. # in approve/deny.
  667. adapter.pause_typing_for_chat(ctx._status_chat_id)
  668. self._close_native_stream_boundary("Approval")
  669. # Redact credentials before display: the raw command string can carry secrets. Both the button and plain-text paths use this value.
  670. cmd = _redact_approval_command(approval_data.get("command", ""))
  671. desc = _redact_approval_command(approval_data.get("description")) or ea_default_reason_text()
  672. flags = {k: approval_data.get(k, d) for k, d in (("allow_permanent", True), ("allow_session", True), ("smart_denied", False))}
  673. # Check the *class*, not the instance — MagicMock auto-creates attributes in tests.
  674. if _renders_exec_approval_buttons(type(adapter)):
  675. try:
  676. fut = self._schedule(
  677. adapter.send_exec_approval(
  678. chat_id=ctx._status_chat_id, command=cmd, session_key=ctx.session_key or "",
  679. description=desc, metadata=ctx._status_thread_metadata,
  680. request_id=approval_data.get("request_id"), **flags,
  681. ),
  682. "send_exec_approval scheduling error",
  683. )
  684. if fut is None:
  685. raise RuntimeError("send_exec_approval: loop unavailable")
  686. outcome = _approval_send_outcome(fut, timeout=15)
  687. if outcome == "sent":
  688. # Without this, a card whose timer runs out keeps live buttons and nobody
  689. # learns the command did NOT run (only the TUI registered a settle hook).
  690. register_timeout_notice(
  691. self, approval_data, command=cmd,
  692. card_message_id=getattr(fut.result(timeout=0), "message_id", None))
  693. return
  694. if outcome == "ambiguous":
  695. # Timeout ≠ failure: the card may have posted with a late ack. The prompt
  696. # registration stays alive so a tap still resolves; re-sending made duplicate
  697. # cards + orphaned "/approve: nothing pending".
  698. logger.warning(
  699. "Button-based approval send timed out — treating "
  700. "as possibly-delivered (no re-send; the prompt "
  701. "stays armed for a late tap)"
  702. )
  703. return
  704. if outcome == "declined":
  705. # P5(b): the connector AUTHORIZED this destination and
  706. # refused it. The text fallback below re-sends the same
  707. # content to the same chat, which would turn a refused
  708. # button card into a delivered plain-text one — the exact
  709. # leak the egress guard exists to stop. A decline is
  710. # definitive, so unlike `ambiguous` the registration is
  711. # torn down; unlike `failed`, nothing is re-sent.
  712. logger.warning(
  713. "Button-based approval DECLINED by the connector's "
  714. "egress guard — not falling back to text (the "
  715. "destination is not approved for this connection)"
  716. )
  717. # RAISE, do not return. This function is the notify_cb for
  718. # `_await_gateway_decision`, which already has a correct
  719. # undeliverable path: a raising notify drops the queue entry
  720. # and returns `notify_failed`, unblocking the tool. Returning
  721. # quietly suppressed the text fallback (right) but left the
  722. # CENTRAL approval entry pending (wrong) — the dangerous
  723. # command then blocked until the approval timeout. My earlier
  724. # comment claimed the registration was torn down; only the
  725. # adapter's private prompt map was.
  726. raise _ExecApprovalDeclined(
  727. "exec approval undeliverable: connector egress declined "
  728. "this destination"
  729. )
  730. logger.warning("Button-based approval failed (send returned error), falling back to text")
  731. except _ExecApprovalDeclined:
  732. # Must escape this handler: the fallback below is a text send to
  733. # the destination the connector just refused.
  734. raise
  735. except Exception as e:
  736. logger.warning("Button-based approval failed, falling back to text: %s", e)
  737. # Plain-text prompt with the adapter's typed prefix (e.g. `!approve`): typed "/" is blocked
  738. # in Slack threads and reserved by Matrix clients.
  739. msg = _format_exec_approval_fallback(cmd, desc, getattr(adapter, "typed_command_prefix", "/"), **flags)
  740. try:
  741. # Mark as approval prompt: WeCom routes it through the control lane and Telegram pushes it
  742. # in "important" mode (#132516). Never ``notify`` — A2A reads that as the turn-final reply.
  743. metadata = {**(ctx._status_thread_metadata or {}), "is_approval_prompt": True}
  744. fut = self._schedule(
  745. adapter.send(ctx._status_chat_id, msg, metadata=_interim_metadata(metadata)), "Approval text-send scheduling error",
  746. )
  747. if fut is not None:
  748. fut.result(timeout=15)
  749. # No card to edit on the text path: the prompt has no buttons to drop and carries
  750. # the /approve instructions, so the timeout notice is posted as a new message.
  751. register_timeout_notice(self, approval_data, command=cmd, card_message_id=None)
  752. except Exception as e:
  753. logger.error("Failed to send approval request: %s", e)
  754. # ── run_sync phases ─────────────────────────────────────────────────────────────────────
  755. def _load_turn_history(self, agent, reused_cached_agent):
  756. from gateway.run import (
  757. _build_gateway_agent_history, _collect_history_media_paths, _message_timestamps_enabled,
  758. _select_cached_agent_history,
  759. )
  760. ctx = self._ctx
  761. from gateway.session_api_turn import api_execution
  762. api = api_execution.get()
  763. if api is not None and api['history'] is not None:
  764. from gateway.run import _collect_history_media_paths
  765. history = api['history']
  766. tail = history[-1] if history else None
  767. if api.get('resume_unanswered') and isinstance(tail, dict) and tail.get('role') == 'user':
  768. # The door's re-run continues the row its failed attempt persisted (#115325). Only a
  769. # bare user tail: a tool suffix is completed work the re-run must still see.
  770. from agent.session_persistence import adopt_unanswered_turn
  771. history = list(history)
  772. adopt_unanswered_turn(history, api['content'], agent)
  773. return history, None, _collect_history_media_paths(history)
  774. # Transcript rows ({role, content, timestamp}) lose timestamps; interrupt-path agent messages
  775. # (tool_calls/tool_call_id/reasoning) pass through intact so the API sees valid assistant→tool
  776. # sequences. Telegram observed=True rows are withheld from replayable history and attached to
  777. # the current addressed message as API-only context.
  778. agent_history, observed_group_context = _build_gateway_agent_history(
  779. ctx.history, channel_prompt=ctx.channel_prompt, inject_timestamps=_message_timestamps_enabled(ctx.user_config),
  780. )
  781. # FTS write-corruption guard: if persistence failed silently the reloaded transcript is stale
  782. # while the SAME cached agent still holds the live conversation (same-session amnesia). Only
  783. # for a reused agent bound to this exact session_id.
  784. # Replacing the live transcript with that shorter copy causes immediate same-session amnesia. See
  785. # #50502.
  786. if reused_cached_agent and getattr(agent, "session_id", None) == ctx.session_id:
  787. selected = _select_cached_agent_history(agent_history, getattr(agent, "_session_messages", None))
  788. # Consecutive lagging turns per session (cleared on /new): one lag is a blip, a streak
  789. # is the #114266 write outage and must escalate past a repeating WARNING.
  790. streaks = getattr(self._runner, "_transcript_lag_streaks", None)
  791. if streaks is None: # tests may build bare runners
  792. streaks = self._runner._transcript_lag_streaks = {}
  793. if selected is not agent_history:
  794. streak = streaks[ctx.session_key] = streaks.get(ctx.session_key, 0) + 1
  795. log = logger.error if streak >= _TRANSCRIPT_LAG_ESCALATION_TURNS else logger.warning
  796. log(
  797. "Persisted transcript lagged live cached history for "
  798. "session %s (disk=%d, memory=%d, consecutive_turns=%d); preserving live "
  799. "conversation context (possible FTS write corruption)%s",
  800. ctx.session_key, len(agent_history), len(selected), streak,
  801. "; state.db is not receiving this session's writes and needs operator attention"
  802. if streak >= _TRANSCRIPT_LAG_ESCALATION_TURNS else "",
  803. )
  804. # The live history bypassed _build_gateway_agent_history's cleanup — re-apply
  805. # the full canonicalization so no replay transform can slip through.
  806. agent_history = canonicalize_replay_history(selected)
  807. else:
  808. streaks.pop(ctx.session_key, None)
  809. # MEDIA paths already in history are excluded from this turn's extraction (compression-safe).
  810. return agent_history, observed_group_context, _collect_history_media_paths(agent_history)
  811. def _prepend_pending_note(self, attr: str) -> None:
  812. """Consume a one-shot per-session note (model switch, /reload-skills) into the NEXT user
  813. message. Nothing hits the transcript out-of-band, so alternation stays intact."""
  814. ctx = self._ctx
  815. notes = getattr(self._runner, attr, None)
  816. note = notes.pop(ctx.session_key, None) if notes and ctx.session_key and ctx.session_key in notes else None
  817. if note:
  818. ctx.message = note + "\n\n" + ctx.message
  819. def _resume_note_interactive(self) -> bool:
  820. """Interactive platforms report the restore and ask what next; event platforms (webhook,
  821. API server) continue the work — nobody is present to answer."""
  822. return bool(getattr(self._runner._delivery_adapter_for(self._ctx.source), "interactive_resume", True))
  823. def _prepare_turn_message(self, agent_history):
  824. """Prepend recovery/notice guidance to ``ctx.message``.
  825. Returns (persist_user_message_override, persist_user_timestamp_override): real user text is
  826. kept separate from API-only recovery guidance so stale guidance never replays as user text.
  827. """
  828. from gateway.run import (
  829. _auto_continue_freshness_window, _is_fresh_gateway_interruption,
  830. _last_transcript_timestamp, _prepare_resume_pending_message, build_resume_recovery_note,
  831. )
  832. ctx = self._ctx
  833. persist_override: Optional[Any] = ctx.persist_user_message
  834. from gateway.session_cli_turn import cli_turn
  835. api_text = cli_turn(ctx.inbound_message_id).get('api_text')
  836. if api_text is not None:
  837. # The classic CLI's model-facing message (voice prefix, its own model-switch and
  838. # skills-reload notes); the admitted text stays the persisted user row.
  839. clean_text = persist_override if persist_override is not None else ctx.message
  840. persist_override = clean_text
  841. ctx.message = api_text
  842. self._prepend_pending_note("_pending_model_notes")
  843. # Auto-continue: history ending with a tool result means the previous turn was cut off
  844. # (restart, crash, SIGTERM). Session-level resume_pending (drain-timeout shutdown) uses
  845. # stronger reason-aware wording that subsumes this case. Both gate on the age of
  846. # ``history[-1]`` (not agent_history, which stripped tool-row timestamps); no stamp = fresh.
  847. window = _auto_continue_freshness_window()
  848. interruption_is_fresh = _is_fresh_gateway_interruption(_last_transcript_timestamp(ctx.history), window_secs=window)
  849. entry = None
  850. if ctx.session_key:
  851. with suppress(Exception):
  852. entry = self._runner.session_store._entries.get(ctx.session_key)
  853. resume_pending = entry is not None and getattr(entry, "resume_pending", False)
  854. resume_reason = (getattr(entry, "resume_reason", None) or "restart_timeout") if resume_pending else None
  855. # resume_pending freshness ALSO uses the restart watchdog's ``last_resume_marked_at`` (the true
  856. # interruption stamp): the transcript clock can be hours older for an active thread, and the
  857. # startup auto-resume turn has empty text, so gating on it alone yields a blank user message.
  858. mark_is_fresh = resume_pending and _is_fresh_gateway_interruption(
  859. getattr(entry, "last_resume_marked_at", None), window_secs=window,
  860. )
  861. if resume_pending and (interruption_is_fresh or mark_is_fresh):
  862. # Empty message = the startup auto-resume turn; there is no NEW user message.
  863. ctx.message, persist_override = _prepare_resume_pending_message(
  864. resume_reason, ctx.message, interactive=self._resume_note_interactive(),
  865. )
  866. elif agent_history and agent_history[-1].get("role") == "tool" and interruption_is_fresh:
  867. persist_override = ctx.message
  868. ctx.message = (
  869. "[System note: A new message has arrived. The conversation "
  870. "history contains pending tool outputs from an interrupted turn. "
  871. "IGNORE those pending results. Address the user's NEW message "
  872. "below FIRST. Do NOT re-execute old tool calls from the history.]\n\n"
  873. + ctx.message
  874. )
  875. self._prepend_pending_note("_pending_skills_reload_notes")
  876. if api_text is not None:
  877. persist_override = clean_text # recovery wording is API-only too, never the stored row
  878. # Safety net: a startup auto-resume event carries empty text; if the resume_pending branch
  879. # did not fire (freshness signals disagreed, marker cleared) we must NOT hand the model a blank
  880. # user turn. Restricted to resume_pending sessions so caption-less image turns are untouched.
  881. if isinstance(ctx.message, str) and not ctx.message.strip() and resume_pending:
  882. ctx.message = build_resume_recovery_note(resume_reason, "", interactive=self._resume_note_interactive())
  883. return persist_override, ctx.persist_user_timestamp
  884. def _native_image_run_message(self):
  885. """Wrap the user turn as an OpenAI-style multimodal content list when
  886. _prepare_inbound_message_text buffered image paths; consume-and-clear so later turns on the
  887. same runner never re-attach stale images. Falls back to plain text when nothing is readable."""
  888. ctx = self._ctx
  889. from gateway.session_api_turn import api_execution
  890. api = api_execution.get()
  891. if api is not None and isinstance(api.get('content'), list):
  892. return api['content']
  893. native_imgs = self._runner._consume_pending_native_image_paths(ctx.session_key)
  894. if not native_imgs:
  895. return ctx.message
  896. try:
  897. from agent.image_routing import build_native_content_parts
  898. parts, skipped = build_native_content_parts(ctx.message, native_imgs)
  899. if skipped:
  900. logger.warning("Native image attachment: skipped %d unreadable path(s): %s", len(skipped), skipped)
  901. if any(p.get("type") == "image_url" for p in parts):
  902. return parts
  903. except Exception as exc:
  904. logger.warning("Native image attachment failed, falling back to text: %s", exc)
  905. return ctx.message
  906. def _run_conversation_with_approval(self, agent, agent_history, observed_group_context,
  907. persist_user_message_override, persist_user_timestamp_override):
  908. """Run the turn with the per-session gateway approval callback registered: dangerous-command
  909. approval blocks the agent thread (mirrors CLI input()); the callback bridges sync→async."""
  910. from gateway.run import _wrap_current_message_with_observed_context
  911. from tools.approval import register_gateway_notify, unregister_gateway_notify
  912. from tools.approval_context import reset_current_session_key, set_current_session_key
  913. ctx = self._ctx
  914. session_key = ctx.session_key or ""
  915. token = set_current_session_key(session_key)
  916. register_gateway_notify(session_key, self._approval_notify_sync)
  917. try:
  918. api_message = _wrap_current_message_with_observed_context(self._native_image_run_message(), observed_group_context)
  919. kwargs = {"conversation_history": agent_history, "task_id": ctx.session_id}
  920. from gateway.session_api_turn import api_execution
  921. api = api_execution.get()
  922. if api is not None and api.get('relay_metadata') and _accepts_keyword(agent.run_conversation, "relay_metadata"):
  923. kwargs["relay_metadata"] = api['relay_metadata'] # main: api_server forwards body.metadata
  924. if _accepts_keyword(agent.run_conversation, "turn_author"):
  925. # Sent on every transport: a provider gating durable writes needs the bot flag in a DM too.
  926. from gateway.session_ingress import admission_author
  927. kwargs["turn_author"] = (api.get('turn_author') if api is not None else
  928. admission_author.get() or {"id": ctx.source.user_id or None, "name": ctx.source.user_name or None,
  929. "is_bot": bool(getattr(ctx.source, "is_bot", False))})
  930. if ctx.title_user_message is not None:
  931. kwargs["title_user_message"] = ctx.title_user_message
  932. if persist_user_message_override is not None:
  933. kwargs["persist_user_message"] = persist_user_message_override
  934. elif observed_group_context:
  935. kwargs["persist_user_message"] = ctx.message
  936. if ctx.persist_user_display_kind:
  937. # Internal self-injected turn: type the persisted user row so UIs render it as a
  938. # timeline notice, not a user bubble (stripped from provider payloads downstream).
  939. kwargs["persist_user_display_kind"] = ctx.persist_user_display_kind
  940. if ctx.persist_user_display_metadata:
  941. kwargs["persist_user_display_metadata"] = ctx.persist_user_display_metadata
  942. if ctx.moa_config is not None:
  943. kwargs["moa_config"] = ctx.moa_config
  944. from gateway.session_onboarding import arm_onboarding_turn
  945. prelude = arm_onboarding_turn(agent, self._approval_owner, agent_history)
  946. if prelude is not None and _accepts_keyword(agent.run_conversation, "prelude"):
  947. kwargs["prelude"] = prelude
  948. if persist_user_timestamp_override is not None:
  949. kwargs["persist_user_timestamp"] = persist_user_timestamp_override
  950. # The RAW inbound id (not event_message_id, the reply anchor) rides the persisted user
  951. # turn so a restart-interrupted turn is recorded WITH its id for drain-window dedup.
  952. if ctx.inbound_message_id is not None:
  953. kwargs["persist_user_platform_id"] = str(ctx.inbound_message_id)
  954. from agent.notification_presentation import notification_turn
  955. from gateway.session_results import execution_result
  956. captured = execution_result.get()
  957. before = (getattr(agent, 'session_prompt_tokens', 0) or 0,
  958. getattr(agent, 'session_completion_tokens', 0) or 0)
  959. with notification_turn(agent, muted=ctx.mute_notification_reply, session_id=ctx.session_id or ""):
  960. from gateway.response_filters import silence_allowed
  961. # Where a bare marker would become the "needed a reply" warning, the agent ends a
  962. # marker that follows a written reply on that reply, before transform/persistence.
  963. agent._silence_marker_needs_reply = not silence_allowed(
  964. ctx.persist_user_display_kind, ctx.reply_expected)
  965. try:
  966. result = agent.run_conversation(api_message, **kwargs)
  967. finally:
  968. agent._silence_marker_needs_reply = False
  969. if captured is not None:
  970. incoming = max(0, (getattr(agent, 'session_prompt_tokens', 0) or 0) - before[0])
  971. outgoing = max(0, (getattr(agent, 'session_completion_tokens', 0) or 0) - before[1])
  972. captured['usage'] = {'input_tokens': incoming, 'output_tokens': outgoing,
  973. 'total_tokens': incoming + outgoing}
  974. _settle_api_runtime_lock(agent, result, captured)
  975. return result
  976. finally:
  977. unregister_gateway_notify(session_key)
  978. # Cancel pending clarify entries so blocked agent threads don't hang past the end of the
  979. # run (interrupt, completion, gateway shutdown). Idempotent.
  980. with suppress(Exception):
  981. from tools.clarify_gateway import clear_session
  982. clear_session(session_key)
  983. reset_current_session_key(token)
  984. def _finish_stream_consumer(self, result, agent_history, stream_consumer):
  985. ctx = self._ctx
  986. # Canonicalize a model-emitted computer-use screenshot path at the common result boundary so
  987. # the streaming finalizer and the non-streaming delivery path see the same response.
  988. if isinstance(result, dict) and isinstance(result.get("final_response"), str):
  989. result["final_response"] = repair_explicit_computer_use_media_paths(
  990. result["final_response"], result.get("messages", []), history_offset=len(agent_history),
  991. )
  992. ctx.result_holder[0] = result
  993. if stream_consumer is None:
  994. return
  995. # Pass final_response as the authoritative finalize payload: it includes post-stream
  996. # augmentation (verifier footer, explainer) the accumulator never saw. Adopt ONLY a genuinely
  997. # completed final: interrupt paths return {interrupted: True, completed: False} with a
  998. # DIAGNOSTIC final_response — adopting it would seal the partial answer over with the
  999. # diagnostic AND suppress the gateway's own error delivery.
  1000. _final_for_stream = None
  1001. if (
  1002. isinstance(result, dict) and not result.get("failed") and not result.get("interrupted")
  1003. and result.get("completed") is not False
  1004. ):
  1005. fr = result.get("final_response")
  1006. if isinstance(fr, str) and fr.strip() and fr != "(empty)":
  1007. _final_for_stream = fr
  1008. if _final_for_stream is None:
  1009. stream_consumer.finish()
  1010. return
  1011. # Duck-type safe: test doubles / older consumers may expose a zero-arg finish().
  1012. try:
  1013. stream_consumer.finish(_final_for_stream)
  1014. except TypeError:
  1015. stream_consumer.finish()
  1016. def _restore_telegram_thread_id_after_split(self, agent_session_id) -> None:
  1017. """Telegram DM whose source.thread_id was lost in the session split (synthetic/recovered
  1018. event): restore it from the binding so _thread_metadata_for_source yields the right
  1019. message_thread_id instead of the General thread (non-fatal)."""
  1020. ctx = self._ctx
  1021. try:
  1022. # run_sync is off-loop (executor); sync DB is fine.
  1023. binding = self._runner._session_db._db.get_telegram_topic_binding_by_session(session_id=agent_session_id)
  1024. if binding and binding.get("thread_id"):
  1025. ctx.source.thread_id = str(binding["thread_id"])
  1026. logger.debug(
  1027. "Restored source.thread_id=%s from binding after session split %s → %s",
  1028. ctx.source.thread_id, ctx.session_id, agent_session_id,
  1029. )
  1030. except Exception:
  1031. logger.debug("Failed to restore thread_id from binding after session split", exc_info=True)
  1032. def _sync_session_after_run(self, agent_history):
  1033. """Sync session_id right after run_conversation(): compression can rotate before a follow-up
  1034. model call fails, and the failure return must still point at the compressed child.
  1035. Returns (compacted_in_place, effective_session_id, effective_history_offset)."""
  1036. ctx = self._ctx
  1037. runner = self._runner
  1038. agent = ctx.agent_holder[0]
  1039. # In-place compaction compacts the transcript WITHOUT rotating the id, so the id-change diff
  1040. # can't see it; compress_context() sets this flag and the gateway re-baselines as for a split.
  1041. compacted_in_place = bool(getattr(agent, "_last_compaction_in_place", False)) if agent else False
  1042. agent_session_id = getattr(agent, 'session_id', ctx.session_id) if agent else ctx.session_id
  1043. session_was_split = bool(agent and ctx.session_key and agent_session_id != ctx.session_id)
  1044. if session_was_split:
  1045. logger.info("Session split detected: %s → %s (compression)", ctx.session_id, agent_session_id)
  1046. entry = runner.session_store._entries.get(ctx.session_key)
  1047. persisted = False
  1048. if entry:
  1049. entry_session_id = getattr(entry, "session_id", None)
  1050. if not ctx._run_still_current():
  1051. logger.info(
  1052. "Skipping session split sync for stale run %s — "
  1053. "generation %s is no longer current",
  1054. ctx.session_key or "?", ctx.run_generation,
  1055. )
  1056. elif entry_session_id == agent_session_id:
  1057. persisted = True
  1058. elif entry_session_id != ctx.session_id:
  1059. logger.info(
  1060. "Skipping session split sync for %s because the "
  1061. "session binding moved from %s to %s before "
  1062. "compression finished",
  1063. ctx.session_key or "?", ctx.session_id, entry_session_id,
  1064. )
  1065. else:
  1066. entry.session_id = agent_session_id
  1067. runner.session_store._save()
  1068. runner.session_store._record_gateway_session_peer(agent_session_id, ctx.session_key, ctx.source)
  1069. persisted = True
  1070. # Only after this run published its split — a stale /stop→/new predecessor must not
  1071. # mutate routing state.
  1072. if persisted:
  1073. src = ctx.source
  1074. if (
  1075. getattr(src, "platform", None) == Platform.TELEGRAM and getattr(src, "chat_type", None) == "dm"
  1076. and getattr(src, "thread_id", None) is None and runner._session_db is not None
  1077. ):
  1078. self._restore_telegram_thread_id_after_split(agent_session_id)
  1079. runner._sync_telegram_topic_binding(src, entry, reason="agent-run-compression")
  1080. from gateway.session_cli_turn import cli_turn
  1081. if not cli_turn(ctx.inbound_message_id).get("model_override"):
  1082. # A `/model --once` / `/moa` turn ran another route for this admission only: the row keeps
  1083. # the session's model, or a /resume would come back on the one-turn model.
  1084. runner._sync_session_model_from_agent(agent_session_id, agent)
  1085. # history_offset=0 whenever the agent's message list lost the original history prefix
  1086. # (split OR in-place compaction): the returned `messages` is the compacted set, persist all
  1087. # of it; slicing past the pre-compaction length would drop everything.
  1088. offset = 0 if (session_was_split or compacted_in_place) else len(agent_history)
  1089. return compacted_in_place, agent_session_id, offset
  1090. def _combined_ephemeral_prompt(self) -> str:
  1091. """Platform context + YAML channel_prompts hint + channel_overrides system_prompt (or global
  1092. ephemeral) + the gateway ephemeral prompt."""
  1093. ctx = self._ctx
  1094. from gateway.session_api_turn import api_execution
  1095. api = api_execution.get()
  1096. if api is not None:
  1097. return api['settings'].get('ephemeral_system_prompt') or ''
  1098. combined = ctx.context_prompt or ""
  1099. for extra in (
  1100. (ctx.channel_prompt or "").strip(),
  1101. self._runner._get_system_prompt_for_channel(
  1102. ctx.source.platform, ctx.source.chat_id or "", thread_id=getattr(ctx.source, "thread_id", None),
  1103. parent_id=getattr(ctx.source, "parent_chat_id", None),
  1104. ),
  1105. ):
  1106. if extra:
  1107. combined = (combined + "\n\n" + extra).strip()
  1108. return combined
  1109. def _append_auto_media_tags(self, final_response: str, result, agent_history, history_media_paths) -> str:
  1110. """Append MEDIA:<path> tags from tool results (e.g. TTS) that the model's final text omits, so
  1111. extract_media() delivers each file once. Scoped to THIS turn (slice at len(agent_history)) so
  1112. a stale MEDIA: path from an earlier turn never rides a later reply; the history-path dedup is
  1113. the secondary guard — and the sole one when mid-run compression shrank the list."""
  1114. from gateway.run import _collect_auto_append_media_tags
  1115. if "MEDIA:" in final_response:
  1116. return final_response
  1117. # Scan tool results for MEDIA:<path> tags that need to be delivered as native audio/file
  1118. # attachments. The TTS tool embeds MEDIA: tags in its JSON response, but the model's final text
  1119. # reply usually doesn't include them. We collect unique tags from tool results and append any that
  1120. # aren't already present in the final response, so the adapter's extract_media() can find and
  1121. # deliver the files exactly once. Scope the scan to THIS turn's tool results only. ``agent_history``
  1122. # was passed into run_conversation as ``conversation_history``, so the agent's returned ``messages``
  1123. # list is ``agent_history`` followed by the messages produced this turn. Slicing at
  1124. # ``len(agent_history)`` isolates the current turn precisely, so a stale MEDIA: path emitted by a
  1125. # tool several turns earlier (still present in the full message list) can never leak onto a later
  1126. # text-only reply. (Fixes #34608) Path-based deduplication against _history_media_paths (collected
  1127. # before run_conversation) is retained as a secondary guard. It is also the sole guard on the
  1128. # fallback branch taken when mid-run context compression shrinks the message list below the original
  1129. # history length, preserving the compression-safe behaviour of #160.
  1130. media_tags, has_voice_directive = _collect_auto_append_media_tags(
  1131. result.get("messages", []), history_offset=len(agent_history), history_media_paths=history_media_paths,
  1132. )
  1133. if not media_tags:
  1134. return final_response
  1135. unique_tags = (["[[audio_as_voice]]"] if has_voice_directive else []) + list(dict.fromkeys(media_tags))
  1136. return final_response + "\n" + "\n".join(unique_tags)
  1137. def run_sync(self):
  1138. from contextlib import ExitStack
  1139. from gateway.session_policy import bind_turn_workspace, policy_for_source, policy_scope, refused_turn_result
  1140. from gateway.session_api_turn import api_policy_scope
  1141. from gateway.session_authorities import active_authority
  1142. from hermes_state_runtime import RuntimeStoreError
  1143. policy = policy_for_source(self._runner, self._ctx.source)
  1144. with ExitStack() as scope:
  1145. try:
  1146. scope.enter_context(policy_scope(policy, authority=active_authority(self._runner)))
  1147. except RuntimeStoreError as exc:
  1148. result = refused_turn_result(policy, exc)
  1149. if result is None:
  1150. raise
  1151. else:
  1152. scope.enter_context(api_policy_scope())
  1153. bind_turn_workspace(policy, self._ctx.session_id)
  1154. result = self._run_sync_scoped()
  1155. from gateway.session_results import execution_result
  1156. captured = execution_result.get()
  1157. if captured is not None:
  1158. cli_fields = self.__dict__.pop("_cli_result_fields", None)
  1159. # A classic CLI viewer reads the run_conversation-shaped dict from the receipt; the
  1160. # delivered reply (message.complete text) keeps the gateway's own copy.
  1161. captured['result'] = (
  1162. {**result, **cli_fields, **({'response_previewed': True} if result.get('response_previewed') else {})}
  1163. if cli_fields else result
  1164. )
  1165. return result
  1166. def _run_sync_scoped(self):
  1167. """Executor-thread body of the turn; returns the gateway result dict.
  1168. The turn message lives on the shared TurnContext (``ctx.message``) so ``_run_agent_inner`` sees
  1169. every rebind. session_key propagates via contextvars (_set_session_env / set_current_session_key)
  1170. — never os.environ["HERMES_SESSION_KEY"], which would misroute approvals across sessions.
  1171. """
  1172. from gateway.run import (
  1173. _current_max_iterations, _normalize_empty_agent_response, _reload_runtime_env_preserving_config_authority,
  1174. _sanitize_gateway_final_response,
  1175. )
  1176. ctx = self._ctx
  1177. runner = self._runner
  1178. # Platform.LOCAL ("local") maps to the "cli" hint key the agent understands.
  1179. # session_key is propagated via contextvars in _set_session_env() (_SESSION_KEY) and via
  1180. # set_current_session_key() (_approval_session_key) below — both concurrency-safe and inherited by
  1181. # tool worker threads. We deliberately do NOT write os.environ["HERMES_SESSION_KEY"] here:
  1182. # os.environ is process-global, so concurrent gateway sessions (e.g. two Discord threads) would
  1183. # clobber each other's value, and a tool thread whose contextvar is unset would fall back to
  1184. # os.environ and read the wrong session key — misrouting command-approval prompts to the wrong
  1185. # thread (#24100). The non-gateway surfaces don't depend on this write: CLI and cron bind the
  1186. # session via contextvars (set_current_session_key / session context), and only the TUI slash-worker
  1187. # *subprocess* exports HERMES_SESSION_KEY (from its own --session-key argv, a separate process) — so
  1188. # removing this in-process gateway write does not affect any of them.
  1189. from gateway.session_policy import policy_for_source
  1190. policy = policy_for_source(runner, ctx.source)
  1191. if policy and policy.yolo and ctx.session_key:
  1192. from tools.approval_yolo import apply_launch_yolo
  1193. apply_launch_yolo(ctx.session_key)
  1194. platform_key = policy.platform if policy else ("cli" if ctx.source.platform == Platform.LOCAL else ctx.source.platform.value)
  1195. combined_ephemeral = self._combined_ephemeral_prompt()
  1196. if policy and policy.skills_prompt:
  1197. combined_ephemeral = "\n\n".join(p for p in (combined_ephemeral, policy.skills_prompt) if p)
  1198. if policy:
  1199. # Rotated .env keys reach this turn's config reads (side tasks read config.yaml), as on the
  1200. # messaging path, where _current_max_iterations() does the same reload; the turn budget
  1201. # stays the session's own. A bypass session never reads the profile.
  1202. if not policy.ignore_user_config:
  1203. _reload_runtime_env_preserving_config_authority()
  1204. max_iterations = policy.max_turns
  1205. else:
  1206. max_iterations = _current_max_iterations()
  1207. from gateway.hosted_room_execution_policy import current_room_execution_policy
  1208. room = current_room_execution_policy()
  1209. if room is not None:
  1210. max_iterations = room.max_iterations
  1211. ctx.enabled_toolsets = list(room.enabled_toolsets)
  1212. try:
  1213. model, runtime_kwargs = runner._resolve_session_agent_runtime(
  1214. source=ctx.source, session_key=ctx.session_key, user_config=ctx.user_config,
  1215. )
  1216. # Stashed by _resolve_session_agent_runtime when the primary's credentials failed and a
  1217. # fallback was resolved before any agent exists (#74349); one-shot per turn.
  1218. pending_fallback_notice = getattr(runner, "_pre_agent_fallback_notice", None)
  1219. runner._pre_agent_fallback_notice = None
  1220. from gateway.session_api_turn import prepare_api_runtime
  1221. model, runtime_kwargs = prepare_api_runtime(model, runtime_kwargs)
  1222. if policy and policy.model and not pending_fallback_notice:
  1223. # The frozen route's model, unless resolution just fell back: the fallback entry's
  1224. # model is the one this agent must send (#112600).
  1225. from gateway.session_authorities import active_authority
  1226. model = policy.current_model(active_authority(runner))
  1227. # The classic CLI's `/model --once` / `/moa`: this admission only, nothing persisted.
  1228. from gateway.session_cli_turn import one_turn_route
  1229. model, runtime_kwargs = one_turn_route(self, policy, model, runtime_kwargs)
  1230. logger.debug(
  1231. "run_agent resolved: model=%s provider=%s session=%s",
  1232. model, runtime_kwargs.get("provider"), ctx.session_key or "",
  1233. )
  1234. except Exception as exc:
  1235. # Model/credential resolution failed before the turn began; the raw text (URLs, status
  1236. # codes) belongs in the log, and the chat gets the commands that fix it. The result is
  1237. # a FAILED turn: a one-shot client exits non-zero, transcript persistence closes the
  1238. # turn, and an API receipt reports failure, never a completed turn with an apology.
  1239. logger.warning("Model resolution failed for session %s: %s", ctx.session_key or "", exc, exc_info=True)
  1240. from agent.turn_failure_copy import stamp_failure
  1241. text, reason, retryable = _resolution_failure_reply(exc, ctx.source.platform)
  1242. return stamp_failure({"final_response": text, "messages": [], "api_calls": 0, "tools": [],
  1243. "failed": True, "completed": False, "error": str(exc)}, reason, retryable)
  1244. from gateway.session_authorities import active_authority
  1245. from gateway.session_worker_construct import session_routing
  1246. pr = session_routing(active_authority(runner), policy) if policy else runner._provider_routing
  1247. reasoning_config = (policy.current_reasoning_config(runner, ctx.session_key) if policy else
  1248. runner._resolve_session_reasoning_config(source=ctx.source, session_key=ctx.session_key, model=model))
  1249. from gateway.session_cli_turn import one_turn_reasoning
  1250. reasoning_config = one_turn_reasoning(self, reasoning_config) # `/model --once --reasoning X`
  1251. runner._service_tier = (policy.current_service_tier(runner, ctx.session_key) if policy else
  1252. runner._resolve_session_service_tier(source=ctx.source, session_key=ctx.session_key))
  1253. from gateway.session_api_turn import api_execution
  1254. api = api_execution.get()
  1255. if api is not None:
  1256. from gateway.platforms.api_server import _request_reasoning_config, _request_service_tier, _REQUEST_OPTION_MISSING
  1257. requested_reasoning = _request_reasoning_config(api['settings'].get('model_options'))
  1258. if requested_reasoning is not None:
  1259. reasoning_config = requested_reasoning
  1260. tier = _request_service_tier(api['settings'].get('model_options'))
  1261. if tier is not _REQUEST_OPTION_MISSING:
  1262. runner._service_tier = tier
  1263. runner._reasoning_config = reasoning_config
  1264. stream_consumer, stream_delta_cb, interim_cb, want_interim = self._setup_stream_consumer(platform_key)
  1265. turn_route = runner._resolve_turn_agent_config(ctx.message, model, runtime_kwargs)
  1266. agent, reused_cached_agent = self._resolve_turn_agent(
  1267. turn_route, platform_key, combined_ephemeral, max_iterations, reasoning_config, pr,
  1268. )
  1269. if pending_fallback_notice:
  1270. # Reuse the in-agent one-shot notice so the pre-agent provider switch is user-visible too.
  1271. agent._pending_fallback_notice = pending_fallback_notice
  1272. self._wire_turn_agent_callbacks(agent, turn_route, reasoning_config, stream_delta_cb, interim_cb, want_interim)
  1273. from gateway.run_turn_progress import local_viewer_turn
  1274. local_sinks = None
  1275. if local_viewer_turn(self) and not ctx.mute_notification_reply:
  1276. # Per turn on the shared cached agent: installed here, restored when the run returns.
  1277. from gateway import local_viewer_events
  1278. local_sinks = local_viewer_events.wire(self, agent)
  1279. from gateway.session_usage import TURN_BASE_ATTR
  1280. setattr(agent, TURN_BASE_ATTR, int(getattr(agent, "session_output_tokens", 0) or 0))
  1281. try:
  1282. agent_history, observed_group_context, history_media_paths = self._load_turn_history(agent, reused_cached_agent)
  1283. persist_msg, persist_ts = self._prepare_turn_message(agent_history)
  1284. compressions = getattr(getattr(agent, "context_compressor", None), "compression_count", None)
  1285. from gateway.session_cli_turn import run_classic_cli_turn
  1286. from gateway.session_secret_prompt import secret_prompt_scope
  1287. from gateway.session_sudo_prompt import sudo_prompt_scope
  1288. from gateway.session_vault_prompt import vault_prompt_scope
  1289. with sudo_prompt_scope(self, agent), secret_prompt_scope(self, agent), vault_prompt_scope(self, agent):
  1290. result = run_classic_cli_turn(self, lambda: self._run_conversation_with_approval(
  1291. agent, agent_history, observed_group_context, persist_msg, persist_ts))
  1292. finally:
  1293. if local_sinks is not None:
  1294. from gateway import local_viewer_events
  1295. local_viewer_events.restore(agent, local_sinks)
  1296. from gateway.session_usage import publish_usage
  1297. publish_usage(self, agent, record=True) # turn end: the final API call's counters
  1298. from gateway.session_local_inventory import publish_settled_info
  1299. publish_settled_info(self, agent)
  1300. persisted_turn = _persisted_turn(agent, agent_history, result, compressions)
  1301. self._finish_stream_consumer(result, agent_history, stream_consumer)
  1302. if _finite_viewer_turn() and getattr(agent, "_codex_session", None) is not None:
  1303. # The classic `chat -q` process exit bounded the codex app-server child; a finite turn
  1304. # hosted here must too, or every one-shot leaves one running until the idle-TTL sweep.
  1305. # Like a fresh process, the next turn respawns it and resumes the stored thread.
  1306. agent._close_codex_session()
  1307. agent._codex_session_prompt = None
  1308. # The streaming-TTS consumer's finish() runs on the outer loop thread after the executor
  1309. # returns, so early run_sync returns are also finalised.
  1310. # See the outer finally/completion section below. See #60671.
  1311. final_response = result.get("final_response")
  1312. # Actual token counts from the agent instance used for this run.
  1313. agent = ctx.agent_holder[0]
  1314. has_comp = bool(agent) and hasattr(agent, "context_compressor")
  1315. comp = agent.context_compressor if has_comp else None
  1316. usage = {
  1317. "last_prompt_tokens": getattr(comp, "last_prompt_tokens", 0) if has_comp else 0,
  1318. "input_tokens": getattr(agent, "session_prompt_tokens", 0) if has_comp else 0,
  1319. "output_tokens": getattr(agent, "session_completion_tokens", 0) if has_comp else 0,
  1320. "model": getattr(agent, "model", None) if agent else None,
  1321. # Most failed results name no provider; the error card's sign-in / update-key action needs it.
  1322. "provider": getattr(agent, "provider", None) if agent else None,
  1323. "context_length": (getattr(comp, "context_length", 0) or 0) if has_comp else 0,
  1324. # The rest of the `-z --usage-file` ledger (hermes_cli/oneshot._USAGE_KEYS): a finite
  1325. # viewer reads the committed result over `prompt.receipt`, so the projection must keep
  1326. # what the in-process one-shot took straight from `run_conversation`.
  1327. **{key: result.get(key) for key in _LEDGER_PASSTHROUGH_KEYS if key in result},
  1328. }
  1329. compacted_in_place, effective_session_id, history_offset = self._sync_session_after_run(agent_history)
  1330. # failure_reason must survive the empty-response path too (TUI billing, transient-failure
  1331. # persistence). compression_deferred (soft lock-contention defer) is distinct from
  1332. # compression_exhausted so the gateway never auto-resets a session a concurrent compressor is
  1333. # about to shrink.
  1334. common = {
  1335. "messages": result.get("messages", []), "api_calls": result.get("api_calls", 0),
  1336. "failed": result.get("failed", False), "failure_reason": result.get("failure_reason"),
  1337. # Usage-window limit (agent/plan_limits.py): the gateway offers the plan's model as a button.
  1338. "usage_limit": result.get("usage_limit"),
  1339. "partial": result.get("partial", False), "completed": result.get("completed"),
  1340. "interrupted": result.get("interrupted", False), "interrupt_message": result.get("interrupt_message"),
  1341. "error": result.get("error"),
  1342. **{key: result[key] for key in _FAILURE_DESCRIPTOR_KEYS if key in result},
  1343. "compression_exhausted": result.get("compression_exhausted", False),
  1344. "compression_deferred": result.get("compression_deferred", False),
  1345. "tools": ctx.tools_holder[0] or [],
  1346. "history_offset": history_offset, "compacted_in_place": compacted_in_place, "session_id": effective_session_id,
  1347. # Same rule as the direct API path (``_finish_turn_result``): the returned `messages` replaced the
  1348. # transcript, so a stored receipt must keep it whole instead of cutting it to this turn's suffix.
  1349. **({"_compressed": True} if compacted_in_place or effective_session_id != ctx.session_id else {}),
  1350. # The loop's proven current-turn row (``export_current_turn_boundary``) for the same `messages`: the
  1351. # stored receipt cuts there instead of re-guessing from text (multimodal or rewritten input + a nudge).
  1352. **({"current_turn_user_idx": result["current_turn_user_idx"]} if "current_turn_user_idx" in result else {}),
  1353. **usage,
  1354. **({"persisted_turn": persisted_turn} if persisted_turn else {}),
  1355. }
  1356. previewed = result.get("response_previewed", False)
  1357. viewer_meta = {}
  1358. if local_sinks is not None:
  1359. was_delivered = getattr(agent, "_interim_text_was_delivered", None)
  1360. previewed = bool(previewed or (callable(was_delivered) and bool(final_response) and was_delivered(final_response) is True))
  1361. viewer_meta = {
  1362. "local_viewer": True,
  1363. **({"billing_block": result["billing_block"]} if isinstance(result.get("billing_block"), dict) else {}),
  1364. **({"warning": result["warning"]} if isinstance(result.get("warning"), str) and result["warning"] else {}),
  1365. }
  1366. if not final_response:
  1367. final_response = _normalize_empty_agent_response(result, final_response or "", history_len=len(agent_history))
  1368. final_response = _sanitize_gateway_final_response(ctx.source.platform, final_response)
  1369. if not final_response:
  1370. final_response = t("gateway.shared.warn_passthrough", error=result["error"]) if result.get("error") else ""
  1371. # NOTE: deliberately omits agent_persisted/response_reused (and last_reasoning outside a local
  1372. # viewer's turn); the caller defaults agent_persisted differently when the key is absent.
  1373. if local_sinks is not None:
  1374. return {
  1375. "final_response": final_response, **common, **viewer_meta,
  1376. "last_reasoning": result.get("last_reasoning"),
  1377. "response_previewed": previewed,
  1378. "response_transformed": bool(result.get("response_transformed")),
  1379. }
  1380. return {"final_response": final_response, **common}
  1381. final_response = self._append_auto_media_tags(final_response, result, agent_history, history_media_paths)
  1382. # Auto-titling runs at TURN START (agent/turn_context.py) from the user's message alone, so a
  1383. # failed/interrupted turn is still titled.
  1384. return {
  1385. "final_response": final_response, "last_reasoning": result.get("last_reasoning"), **common, **viewer_meta,
  1386. "response_previewed": previewed,
  1387. "response_transformed": result.get("response_transformed", False),
  1388. "response_reused": result.get("response_reused", False),
  1389. # Lets the persistence block tell whether the codex app-server path self-persisted (it
  1390. # didn't — see codex_runtime.py); default True keeps skip-db for the standard runtime.
  1391. "agent_persisted": result.get("agent_persisted", True),
  1392. }