run_turn.py 131 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239
  1. """Agent-turn execution for GatewayRunner: _handle_message_with_agent, _run_agent*, proxy path,
  2. MCP reload. Bound onto ``GatewayRunner`` via the MRO; ``gateway.run`` internals
  3. are imported lazily inside method bodies (import cycle) so ``patch("gateway.run.X")`` still works.
  4. """
  5. from __future__ import annotations
  6. from pm import install_hint
  7. import logging
  8. from typing import TYPE_CHECKING
  9. import asyncio
  10. import dataclasses
  11. import inspect
  12. import json
  13. import os
  14. import queue
  15. import threading
  16. import time
  17. from agent.i18n import t
  18. from agent.session_activity import format_iteration_progress
  19. from agent.turn_failure_copy import FAILED_TURN_DISPLAY_KIND, FAILED_TURN_NOTICE, PARTIAL_FAILED_TURN_NOTICE
  20. from contextlib import nullcontext, suppress
  21. from contextvars import copy_context
  22. from gateway.bot_peer_notices import automatic_notices_allowed
  23. from gateway.config import Platform
  24. from gateway.platforms.base import BasePlatformAdapter, ProcessingOutcome
  25. from gateway.platforms.event import MessageEvent
  26. from gateway.platforms.helpers import typing_metadata_for_turn
  27. from gateway.response_filters import (
  28. display_kind_for_event, display_metadata_for_event, is_machinery_display_kind, persisted_user_metadata_for_event,
  29. reply_expected_metadata,
  30. )
  31. from gateway.warning_notifications import diagnostic_metadata, diagnostic_turn_muted, diagnostic_wake_muted
  32. from gateway.session import (
  33. SessionSource, _session_key_namespace, build_channel_continuity_note,
  34. build_session_context,
  35. )
  36. from gateway.session_transcript import TranscriptReadError
  37. from gateway.turn_context import TurnContext
  38. from gateway.turn_lease import DEFAULT_LEASE_WAIT, TurnLeaseTimeoutError
  39. from hermes_constants import get_hermes_home_override
  40. from pathlib import Path
  41. from typing import Any, Callable, Dict, List, Optional, Tuple
  42. from utils import base_url_hostname
  43. if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
  44. from gateway.run import GatewayRunner
  45. from gateway.run_turn_runner import TurnRunner
  46. # Log-record parity with the origin module.
  47. logger = logging.getLogger("gateway.run")
  48. _tool_call_logger_lock = threading.Lock()
  49. def _tool_call_logger() -> logging.Logger:
  50. """Process-wide ``hermes.tool_calls`` Logger + one RotatingFileHandler on logs/tool_calls.log.
  51. Named Loggers live in ``logging.Logger.manager.loggerDict`` forever, so the former per-turn name
  52. (``hermes.tool_calls.<id(log_queue)>``) leaked one Logger per logged turn (#62950); a single
  53. shared handler also keeps concurrent turns from double-writing lines."""
  54. tool_logger = logging.getLogger("hermes.tool_calls")
  55. with _tool_call_logger_lock:
  56. if not tool_logger.handlers:
  57. from logging.handlers import RotatingFileHandler
  58. from agent.redact import RedactingFormatter
  59. from gateway.run import _hermes_home
  60. log_dir = _hermes_home / "logs"
  61. log_dir.mkdir(parents=True, exist_ok=True)
  62. handler = RotatingFileHandler(
  63. log_dir / "tool_calls.log", maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8",
  64. )
  65. handler.setFormatter(RedactingFormatter("%(message)s"))
  66. tool_logger.setLevel(logging.INFO)
  67. tool_logger.propagate = False
  68. tool_logger.addHandler(handler)
  69. return tool_logger
  70. _CONTEXT_OVERFLOW_ERROR_PHRASES = (
  71. "context length", "context size", "context window",
  72. "maximum context", "token limit", "too many tokens",
  73. "reduce the length", "exceeds the limit",
  74. "request entity too large", "prompt is too long",
  75. "payload too large", "input is too long",
  76. )
  77. def _unexpected_silence_reply() -> str:
  78. """Reply when the model returned only a silence marker for a message that needed an answer."""
  79. return t("gateway.errors.unexpected_silence")
  80. def is_context_overflow_failure_result(agent_result: dict, history_len: int) -> bool:
  81. """One verdict for "this failed turn is a context overflow", shared by transcript persistence
  82. (#1630 skip) and the user-facing reply so the two can never disagree.
  83. Multi-word phrases (not bare "exceed"/"token") avoid matching "rate limit exceeded" or
  84. "invalid authentication token"; a bare 400 only counts on a long session."""
  85. if not agent_result.get("failed"):
  86. return False
  87. if agent_result.get("compression_exhausted"):
  88. return True
  89. err = str(agent_result.get("error") or "").lower()
  90. return any(p in err for p in _CONTEXT_OVERFLOW_ERROR_PHRASES) or ("400" in err and history_len > 50)
  91. # Setup/prefix rows rather than conversation: the agent rebuilds its own system prompt, and a
  92. # transcript meta row is logging-only — neither reaches the model, but both are the head a
  93. # fail-closed payload keeps.
  94. _HYGIENE_SETUP_ROLES = ("system", "session_meta")
  95. def bound_model_input_without_hygiene(history: list[Any], limit: int) -> list[Any]:
  96. """Fail-closed in-context bound for a turn where hygiene has not landed (#111988).
  97. Keeps the leading ``system``/``session_meta`` setup rows plus the newest tail, total <= ``limit``.
  98. Deterministic (the same transcript always yields the same cut) and payload-only: the stored
  99. transcript is never touched, so the agent's durable-prefix slice (``history_offset``) is
  100. unaffected. Returns ``history`` unchanged — same object — when nothing needs dropping, so the
  101. landed-compression and below-the-limit paths stay byte-identical.
  102. """
  103. if len(history) <= limit:
  104. return history
  105. head_end = 0
  106. while (head_end < len(history) and isinstance(history[head_end], dict)
  107. and history[head_end].get("role") in _HYGIENE_SETUP_ROLES):
  108. head_end += 1
  109. # Always leave room for the newest row: a setup-only payload would answer nothing.
  110. head_end = min(head_end, limit - 1)
  111. tail_start = len(history) - (limit - head_end)
  112. # Never start the kept tail on a tool result: its parent assistant(tool_calls) row is dropped
  113. # with it, and an orphaned tool result is an invalid sequence for every provider.
  114. while (tail_start < len(history) and isinstance(history[tail_start], dict)
  115. and history[tail_start].get("role") == "tool"):
  116. tail_start += 1
  117. return history[:head_end] + history[tail_start:]
  118. def hygiene_no_commit_reason(agent) -> str:
  119. """Name WHY a hygiene compression left the session id unchanged with no in-place commit.
  120. The terminal ``else`` used to blame "no session_db on the hygiene agent" for every route into it,
  121. but that is one of several causes (#71097): an attempt that ABORTED before any commit boundary
  122. (lock skip, transient cooldown, summary timeout, codex thread interrupted) leaves
  123. ``_last_compression_attempt_in_place`` at ``None``; a DB-less agent is only the case when
  124. ``_session_db`` really is missing. Read the per-attempt signals the compressor sets, in that order."""
  125. if not bool(getattr(agent, "_last_compression_attempt_recorded", False)):
  126. return "compression did not run"
  127. lock_skip = getattr(agent, "_compression_skipped_due_to_lock", None)
  128. if lock_skip is True or isinstance(lock_skip, str):
  129. return "attempt skipped: compression lease held by another process"
  130. blocked = getattr(agent, "_compression_blocked_transient", None)
  131. if blocked:
  132. return f"attempt blocked: {blocked}"
  133. if getattr(agent, "_last_compression_attempt_in_place", None) is None:
  134. detail = "summary timed out" if getattr(agent, "_last_compression_timed_out", False) else "aborted before commit"
  135. warning = getattr(agent, "_last_compression_summary_warning", None)
  136. return f"attempt {detail}" + (f": {warning}" if warning else "")
  137. if getattr(agent, "_session_db", None) is None:
  138. return "no session_db on the hygiene agent"
  139. return "in-place commit did not complete"
  140. from gateway.run_turn_prepare import GatewayTurnPrepareMixin
  141. from gateway.run_background import GatewayBackgroundMixin
  142. from gateway.run_turn_hygiene import GatewayTurnHygieneMixin
  143. from gateway.run_turn_persistence import GatewayTurnPersistenceMixin
  144. class GatewayTurnMixin(GatewayBackgroundMixin, GatewayTurnPrepareMixin, GatewayTurnHygieneMixin, GatewayTurnPersistenceMixin):
  145. """Agent-turn execution for GatewayRunner (see module docstring)."""
  146. def _event_thread_metadata(self, event, source):
  147. """Thread metadata for a send that replies to ``event`` on ``source``."""
  148. return self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))
  149. @staticmethod
  150. def _pop_post_delivery_callback(adapter, key, generation):
  151. """Pop the adapter's deferred post-delivery callback for ``key`` (legacy dict fallback)."""
  152. if getattr(type(adapter), "pop_post_delivery_callback", None) is not None:
  153. return adapter.pop_post_delivery_callback(key, generation=generation)
  154. if adapter and hasattr(adapter, "_post_delivery_callbacks"):
  155. return adapter._post_delivery_callbacks.pop(key, None)
  156. return None
  157. async def _hmwa_stop_typing_for_turn(self, event, source):
  158. """Stop the typing indicator (never raises), with the delivery path's thread/workspace routing
  159. metadata (Slack AI status is thread-scoped), on this turn's own message, not the chat's latest."""
  160. from gateway.platforms.helpers import typing_metadata_for_turn
  161. with suppress(Exception):
  162. _typing_adapter = self._delivery_adapter_for(source)
  163. _kind = type(_typing_adapter)
  164. if _typing_adapter and callable(getattr(_kind, "_stop_typing_with_metadata", None)):
  165. await _typing_adapter._stop_typing_with_metadata(source.chat_id, typing_metadata_for_turn(_typing_adapter,
  166. source.chat_id, self._reply_anchor_for_event(event), self._event_thread_metadata(event, source)))
  167. elif _typing_adapter and callable(getattr(_kind, "stop_typing", None)):
  168. await _typing_adapter.stop_typing(source.chat_id)
  169. async def _handle_message_with_agent(self, event, source, _quick_key: str, run_generation: int):
  170. """Inner handler that runs under the _running_agents sentinel guard."""
  171. _msg_start_time = time.time()
  172. _platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform)
  173. logger.info(
  174. "inbound message: platform=%s user=%s chat=%s msg=%r reply_to_id=%s reply_to_text=%r",
  175. _platform_name, source.user_name or source.user_id or "unknown",
  176. source.chat_id or "unknown", (event.text or "")[:80].replace("\n", " "),
  177. getattr(event, "reply_to_message_id", None),
  178. (getattr(event, "reply_to_text", None) or "")[:80].replace("\n", " "),
  179. )
  180. resolved = await self._hmwa_resolve_session(event, source)
  181. if resolved is None:
  182. return
  183. source, session_entry, session_key = resolved
  184. prepared, _session_env_tokens = await self._hmwa_prepare_turn(
  185. event, source, session_entry, session_key, _quick_key, run_generation,
  186. )
  187. if not isinstance(prepared, self._PreparedTurn):
  188. return prepared
  189. history, message_text = prepared.history, prepared.message_text
  190. try:
  191. hook_ctx = {
  192. "platform": source.platform.value if source.platform else "",
  193. "user_id": source.user_id,
  194. "chat_id": source.chat_id or "",
  195. "thread_id": str(source.thread_id) if getattr(source, "thread_id", None) else "",
  196. "chat_type": getattr(source, "chat_type", "") or "",
  197. "session_id": session_entry.session_id,
  198. "message": message_text[:500],
  199. }
  200. await self.hooks.emit("agent:start", hook_ctx)
  201. # Capture the launch session id so post-run compression publication is identity-guarded
  202. # (a /new may move session_entry.session_id while the old run is still unwinding).
  203. from gateway.run_heartbeat_acceptance import heartbeat_owner_is_current
  204. if not heartbeat_owner_is_current(self, event, session_key):
  205. return
  206. _run_start_session_id = session_entry.session_id
  207. _turn_started_monotonic = time.monotonic()
  208. # Admission/typing is not execution. All routing, authorization and
  209. # turn preparation gates have passed when the agent runner is entered.
  210. event._heartbeat_execution_started = True
  211. _turn_channel_prompt, _turn_source = self._pinned_channel_inputs( # internal/synthetic reuse pins
  212. session_key, event.channel_prompt, source,
  213. internal=event.internal or getattr(event, "inherit_channel_pin", False))
  214. if not event.internal:
  215. # Persist the coherent context+channel pair before execution: a crash during the
  216. # human turn may be followed by an internal startup-resume on the next process.
  217. await self._persist_prompt_pins(session_key, _run_start_session_id)
  218. agent_result = await self._run_agent(
  219. message=message_text, context_prompt=prepared.context_prompt, history=history, source=_turn_source,
  220. session_id=_run_start_session_id, session_key=session_key,
  221. run_generation=run_generation, event_message_id=self._reply_anchor_for_event(event),
  222. inbound_message_id=str(event.message_id) if event.message_id else None,
  223. channel_prompt=_turn_channel_prompt, moa_config=getattr(event, "_moa_config", None),
  224. title_user_message=prepared.title_user_message,
  225. persist_user_message=prepared.persist_user_message,
  226. persist_user_timestamp=prepared.persist_user_timestamp,
  227. persist_user_display_kind=prepared.persist_user_display_kind,
  228. reply_expected=event.reply_expected,
  229. persist_user_display_metadata={
  230. "gateway_input_owner": prepared.persistence_owner, **persisted_user_metadata_for_event(event),
  231. **reply_expected_metadata(event.reply_expected), **diagnostic_metadata(event)},
  232. message_type=event.message_type,
  233. scheduled_heartbeat=bool(getattr(event, "_heartbeat_session_id", None)),
  234. )
  235. _turn_seconds = time.monotonic() - _turn_started_monotonic
  236. # A queued (/queue) chain answered the LAST message of the chain, so the outer final
  237. # send (bracketed by the adapter against this event) must be ledgered under that
  238. # message's id or it collides with an earlier turn's row carrying the same text. Reply
  239. # routing is untouched: the anchor still comes from this event.
  240. if isinstance(agent_result, dict):
  241. _terminal_inbound = agent_result.get("queued_terminal_inbound_id")
  242. if _terminal_inbound:
  243. event.ledger_message_id = str(_terminal_inbound)
  244. if "queued_terminal_notification_category" in agent_result:
  245. event.metadata["notification_category"] = agent_result["queued_terminal_notification_category"]
  246. if isinstance(agent_result.get("_notification_reply_muted"), bool):
  247. event._notification_reply_muted = agent_result["_notification_reply_muted"]
  248. await self._hmwa_stop_typing_for_turn(event, source)
  249. if not self._is_session_run_current(_quick_key, run_generation):
  250. self._hmwa_discard_stale_result(source, _quick_key, run_generation)
  251. return None
  252. response, _intentional_silence, agent_messages = await self._hmwa_shape_agent_response(
  253. agent_result, source, history, session_entry, session_key,
  254. _quick_key, run_generation, _run_start_session_id, _platform_name, _msg_start_time,
  255. persist_user_display_kind=prepared.persist_user_display_kind,
  256. reply_expected=event.reply_expected,
  257. )
  258. response = self._usage_window_chat_wording(agent_result, self._hmwa_prepend_reasoning(agent_result, response, source, _intentional_silence))
  259. _footer_line = self._hmwa_runtime_footer_line(agent_result, source, _turn_seconds)
  260. # Streaming already delivered the body: the footer goes out as a trailing send instead.
  261. if _footer_line and response and not agent_result.get("already_sent") and not _intentional_silence:
  262. response = f"{response}\n\n{_footer_line}"
  263. await self._hmwa_post_turn_hooks(hook_ctx, agent_result, response)
  264. agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure = (
  265. self._hmwa_classify_turn_failure(agent_result, history, session_entry)
  266. )
  267. if agent_failed_early and not is_context_overflow_failure and automatic_notices_allowed(source):
  268. response = self._hmwa_add_failed_turn_notice(response, self._hmwa_failed_turn_notice(agent_result))
  269. response, session_entry = await self._hmwa_compression_exhaustion_reset(
  270. agent_result, response, session_entry, session_key, source,
  271. )
  272. await self._hmwa_persist_turn_transcript(
  273. event=event, source=source, session_entry=session_entry, session_key=session_key,
  274. agent_result=agent_result, agent_messages=agent_messages, prepared=prepared,
  275. response=response, agent_failed_early=agent_failed_early,
  276. hidden_reasoning_incomplete=hidden_reasoning_incomplete,
  277. is_context_overflow_failure=is_context_overflow_failure,
  278. )
  279. await self._record_room_silence_decision(
  280. session_key, agent_result, _intentional_silence,
  281. display_kind=prepared.persist_user_display_kind, reply_expected=event.reply_expected)
  282. return await self._deliver_turn_response_then_limit_offer(
  283. event, source, session_entry, session_key, run_generation,
  284. agent_result, agent_messages, response, _footer_line, _intentional_silence,
  285. )
  286. except Exception as e:
  287. return await self._hmwa_agent_error_reply(e, event, source, session_entry, session_key, prepared)
  288. finally:
  289. # Restore session context variables to their pre-handler state
  290. self._clear_session_env(_session_env_tokens)
  291. def _profile_scope_for_source(self, source: SessionSource):
  292. """``_profile_runtime_scope`` for ``source``'s profile when a secret scope is required.
  293. Under multiplexing config/skills/memory resolve to the source profile's home AND credentials
  294. come from its secret scope (never process-global ``os.environ``). A standalone gateway
  295. (``multiplex_profiles`` off) still binds once a hosted room has flipped the process-wide
  296. credential guard — see ``_standalone_launch_scope``."""
  297. from gateway.run import _profile_runtime_scope
  298. from gateway.session_policy import local_desktop_turn
  299. home = self._profile_scope_key_for_source(source)
  300. return local_desktop_turn(self, source, _profile_runtime_scope(home) if home is not None
  301. else self._standalone_launch_scope())
  302. def _profile_scope_key_for_source(self, source: SessionSource) -> Optional[Path]:
  303. """Profile home ``_profile_scope_for_source`` binds for ``source``, or ``None`` when it falls
  304. back to the standalone launch scope. The single owner of that branch condition: callers that
  305. group work per scope (heartbeat restore) key on this so they cannot drift from the scope
  306. actually entered."""
  307. if getattr(getattr(self, "config", None), "multiplex_profiles", False):
  308. return self._resolve_profile_home_for_source(source)
  309. return None
  310. def _async_profile_scope_for_source(self, source: SessionSource):
  311. """``async with`` twin of :meth:`_profile_scope_for_source` (secret hydration off-loop).
  312. Slash dispatch runs under the RECEIVING bot's scope (auth needs its ``.env``), which is not
  313. the routed runtime when a bot serves another profile's chat; every handler reading
  314. home-relative state (pending writes, memory store, config) binds the runtime here (#119915)."""
  315. from gateway.run import _async_profile_runtime_scope
  316. home = self._profile_scope_key_for_source(source)
  317. if home is not None:
  318. return _async_profile_runtime_scope(home)
  319. from tui_gateway.launch_profile_policy import async_launch_profile_scope_if_multiplexed
  320. return async_launch_profile_scope_if_multiplexed()
  321. @staticmethod
  322. def _standalone_launch_scope():
  323. """Scope for a standalone gateway's own (launch-profile) work: a no-op until the process hosts
  324. another profile home, then the launch profile's OWN runtime scope.
  325. A native hosted room running a second profile calls
  326. ``tui_gateway.launch_profile_policy.activate_multi_profile_hosting`` inside the gateway process,
  327. so ``get_secret`` fails closed for every unscoped read afterwards — including the standalone
  328. gateway's ordinary turns, which never bound a scope because ``multiplex_profiles`` is off
  329. (#112878). The launch profile is a profile too: bind its ``.env`` over the env frozen at
  330. activation (a key injected by systemd / ``op run`` has no file to rebuild it from), never a
  331. secondary's scope and never live ``os.environ``."""
  332. from tui_gateway.launch_profile_policy import launch_profile_scope_if_multiplexed
  333. return launch_profile_scope_if_multiplexed()
  334. def _media_delivery_scope_for_source(self, source: SessionSource):
  335. """Home + terminal-policy scope for validating a turn's MEDIA / local-file paths on the
  336. adapter's delivery side, which runs after the routed turn scope was reset.
  337. Docker path translation (``platforms/base.py::_translate_docker_container_media_path``)
  338. infers the producing container from the ACTIVE profile (``get_active_profile_name``) and the
  339. scope-aware ``TERMINAL_DOCKER_VOLUMES``; without this a secondary's ``MEDIA:/output/x.png``
  340. resolves against the default profile's sandbox and mounts (#109024). No secret hydration:
  341. path validation reads no credentials and this runs on the event loop."""
  342. if not getattr(getattr(self, "config", None), "multiplex_profiles", False):
  343. return nullcontext()
  344. from gateway.run import _profile_runtime_scope
  345. return _profile_runtime_scope(self._resolve_profile_home_for_source(source), {})
  346. def _reset_notice_session_info(self, source: SessionSource) -> str:
  347. """Session-info block for the auto-reset notice, resolved inside the profile serving ``source``.
  348. Call via ``asyncio.to_thread``: resolution can block (credential refresh, context-length
  349. probes), and the scope is entered here so contextvars behave in the worker thread."""
  350. with self._profile_scope_for_source(source):
  351. return self._format_session_info()
  352. def _format_session_info(self) -> str:
  353. """Model / provider / context-length / endpoint block so users can spot bad context detection."""
  354. from gateway.run import _resolve_gateway_model_context
  355. resolved = _resolve_gateway_model_context()
  356. context_length = resolved.context_length
  357. ctx_source = {
  358. "config": "config",
  359. "default": "fallback, not a detected limit — set model.context_length in config to override",
  360. }.get(resolved.context_source, "detected")
  361. ctx_display = (
  362. f"{context_length / 1_000_000:.1f}M" if context_length >= 1_000_000
  363. else f"{context_length // 1_000}K" if context_length >= 1_000 else str(context_length)
  364. )
  365. lines = [
  366. t("gateway.session.info_model", model=resolved.model),
  367. t("gateway.session.info_provider", provider=resolved.provider or "openrouter"),
  368. t("gateway.session.info_context", tokens=ctx_display, source=ctx_source),
  369. ]
  370. if (resolved.provider or "") == "moa":
  371. # The preset name hides who pays: the aggregator runs every tool-loop step (#112359).
  372. from hermes_cli.config import load_config
  373. from hermes_cli.moa_config import normalize_moa_config
  374. agg = normalize_moa_config(load_config().get("moa"))["presets"].get(resolved.model, {}).get("aggregator") or {}
  375. if agg:
  376. lines.append(t("gateway.session.info_acting_model", provider=agg.get("provider"), model=agg.get("model")))
  377. base_url = resolved.base_url
  378. if base_url and base_url_hostname(base_url) in ("localhost", "127.0.0.1", "0.0.0.0"):
  379. lines.append(t("gateway.session.info_endpoint", url=base_url))
  380. return "\n".join(lines)
  381. def _resolve_enabled_toolsets_for_source(
  382. self, user_config: dict, source: SessionSource, platform_key: str,
  383. ) -> list:
  384. """Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override
  385. validated through the SAME ``_get_platform_tools`` path (unknown / platform-restricted
  386. toolsets dropped, not trusted)."""
  387. from hermes_cli.tools_config import _get_platform_tools
  388. try:
  389. adapter = self._delivery_adapter_for(source)
  390. override = adapter.toolsets_for_source(source) if adapter is not None else None
  391. except Exception:
  392. override = None
  393. if override and isinstance(override, list):
  394. pts = dict(user_config.get("platform_toolsets") or {})
  395. pts[platform_key] = [str(x) for x in override]
  396. user_config = {**user_config, "platform_toolsets": pts}
  397. return sorted(_get_platform_tools(user_config, platform_key))
  398. def _resolve_turn_toolsets(self, user_config: dict, source: SessionSource, platform_key: str):
  399. """``(enabled_toolsets, disabled_toolsets)`` for an agent run on ``source``."""
  400. from agent.skill_utils import parse_config_string_list
  401. enabled = self._resolve_enabled_toolsets_for_source(user_config, source, platform_key)
  402. disabled = parse_config_string_list((user_config.get("agent") or {}).get("disabled_toolsets")) or None
  403. return enabled, disabled
  404. def _mcp_reload_refresh_cached_agents(self, multiplex: bool, profile) -> None:
  405. """Refresh cached agents so existing sessions see new MCP tools on their next turn without
  406. a history-destroying ``/new``. Each agent keeps its build-time toolset selection EXACTLY: a
  407. session built with restricted enabled_toolsets (e.g. ["safe"]) must NOT silently gain tools."""
  408. try:
  409. from tools.mcp_tool_agent import refresh_agent_mcp_tools
  410. _cache = getattr(self, "_agent_cache", None)
  411. _cache_lock = getattr(self, "_agent_cache_lock", None)
  412. if _cache_lock is None or not _cache:
  413. return
  414. # Multiplex: only this profile's sessions (another profile's agent would get this registry).
  415. _ns_prefix = _session_key_namespace(profile) + ":" if multiplex else None
  416. with _cache_lock:
  417. for _sess_key, _entry in list(_cache.items()):
  418. if _ns_prefix and not str(_sess_key).startswith(_ns_prefix):
  419. continue
  420. _agent = _entry[0] if isinstance(_entry, tuple) else _entry
  421. if _agent is not None:
  422. refresh_agent_mcp_tools(_agent, quiet_mode=True)
  423. except Exception as _exc:
  424. logger.debug("Failed to update cached agent tools after MCP reload: %s", _exc)
  425. async def _execute_mcp_reload(self, event: MessageEvent) -> str:
  426. """Disconnect, reconnect, and notify MCP tool changes (shared by button / text / no-confirm paths).
  427. Under multiplex the reload runs inside the requesting profile's runtime scope (entered here
  428. when the caller did not) and only that profile's servers are torn down and rediscovered.
  429. See #95518.
  430. """
  431. from gateway.run import _profile_runtime_scope
  432. multiplex = bool(getattr(self.config, "multiplex_profiles", False))
  433. if multiplex and not get_hermes_home_override():
  434. profile_home = self._resolve_profile_home_for_source(event.source)
  435. with _profile_runtime_scope(Path(profile_home)):
  436. return await self._execute_mcp_reload(event)
  437. try:
  438. from tools.mcp_tool_lifecycle import shutdown_mcp_servers
  439. from tools.mcp_tool_discovery import discover_mcp_tools
  440. from tools.mcp_tool import _servers, _lock, _server_visible_in_scope
  441. from tools.mcp_tool_agent import reprobe_tool_availability
  442. from tools.mcp_tool_scope import _key_name
  443. from tools.registry import registry
  444. reload_scope = registry.current_scope_key() if multiplex else None
  445. def _scoped_server_names() -> set:
  446. with _lock:
  447. return {_key_name(key) for key in _servers if _server_visible_in_scope(key, reload_scope)}
  448. old_servers = _scoped_server_names()
  449. await self._run_in_executor_with_context(lambda: shutdown_mcp_servers(scope=reload_scope))
  450. # Explicit reload also re-probes tool availability (check_fn).
  451. reprobe_tool_availability()
  452. # Reconnect by discovering tools (reads config.yaml fresh). A chat command cannot finish
  453. # a browser OAuth flow either: an expired token parks with a `hermes mcp login` hint.
  454. from tools.mcp_oauth import suppress_interactive_oauth
  455. with suppress_interactive_oauth():
  456. new_tools = await self._run_in_executor_with_context(discover_mcp_tools)
  457. connected_servers = _scoped_server_names()
  458. if reload_scope is not None:
  459. from tools.mcp_tool import _mcp_tool_server_names
  460. with _lock:
  461. new_tools = [n for n in new_tools if _mcp_tool_server_names.get(n) in connected_servers]
  462. # (label, i18n key, names); i18n lines list reconnected first, the injected note added first.
  463. changes = (
  464. ("Reconnected", "gateway.reload_mcp.reconnected", connected_servers & old_servers),
  465. ("Added", "gateway.reload_mcp.added", connected_servers - old_servers),
  466. ("Removed", "gateway.reload_mcp.removed", old_servers - connected_servers),
  467. )
  468. lines = [t("gateway.reload_mcp.header")] + [
  469. t(key, names=", ".join(sorted(names))) for _label, key, names in changes if names
  470. ]
  471. from tools.mcp_tool_discovery import mcp_servers_not_connected
  472. failed = await self._run_in_executor_with_context(mcp_servers_not_connected)
  473. lines.append(t("gateway.reload_mcp.none_connected") if not connected_servers else t(
  474. "gateway.reload_mcp.tools_available", tools=len(new_tools), servers=len(connected_servers)))
  475. if failed: # a configured server that did not connect is named, never passed over as success
  476. lines.append(t("gateway.reload_mcp.failed", error=f"did not connect: {', '.join(failed)}"))
  477. self._mcp_reload_refresh_cached_agents(multiplex, event.source.profile)
  478. # Append a note at the END of the history (preserves the prompt-cache prefix).
  479. change_parts = [
  480. f"{label} servers: {', '.join(sorted(names))}"
  481. for label, _key, names in (changes[1], changes[2], changes[0]) if names
  482. ]
  483. tool_summary = f"{len(new_tools)} MCP tool(s) now available" if new_tools else "No MCP tools available"
  484. change_detail = ". ".join(change_parts) + ". " if change_parts else ""
  485. reload_msg = {
  486. "role": "user",
  487. "content": f"[IMPORTANT: MCP servers have been reloaded. {change_detail}{tool_summary}. The tool list for this conversation has been updated accordingly.]",
  488. }
  489. with suppress(Exception): # Best-effort; don't fail the reload over a transcript write
  490. session_entry = await self.async_session_store.get_or_create_session(event.source)
  491. await self.async_session_store.append_to_transcript(session_entry.session_id, reload_msg)
  492. return "\n".join(lines)
  493. except Exception as e:
  494. logger.warning("MCP reload failed: %s", e)
  495. return t("gateway.reload_mcp.failed", error=e)
  496. def _get_proxy_url(self) -> Optional[str]:
  497. """Proxy URL if proxy mode is configured (GATEWAY_PROXY_URL env wins over ``gateway.proxy_url``).
  498. Per-profile like GATEWAY_PROXY_KEY: under multiplex a raw environ read would ship a secondary's
  499. turns (authenticated with ITS scoped key) to the default profile's proxy. Same fallback shape as
  500. the key — only ``UnscopedSecretError`` (the unscoped default-profile path) reads the env."""
  501. from gateway.run import _load_gateway_config
  502. from agent.secret_scope import UnscopedSecretError, get_secret
  503. try:
  504. url = (get_secret("GATEWAY_PROXY_URL") or "").strip()
  505. except UnscopedSecretError:
  506. url = os.getenv("GATEWAY_PROXY_URL", "").strip()
  507. if not url:
  508. url = ((_load_gateway_config().get("gateway") or {}).get("proxy_url") or "").strip()
  509. return url.rstrip("/") if url else None
  510. def _build_stream_consumer_config(
  511. self, source: SessionSource, scfg: Any, adapter: Any, *, on_missing_cursor: str,
  512. ) -> tuple[Any, Optional[Callable[[], None]]]:
  513. """Build the shared ``StreamConsumerConfig`` and optional Telegram pause-typing closure.
  514. For non-editing adapters ``on_missing_cursor="fallback"`` streams with an empty cursor;
  515. ``"raise"`` raises ``RuntimeError`` so the caller skips streaming entirely."""
  516. from gateway.stream_consumer import StreamConsumerConfig
  517. _pause_typing_before_finalize = None
  518. if source.platform == Platform.TELEGRAM and hasattr(adapter, "pause_typing_for_chat"):
  519. def _pause_typing_before_finalize(_adapter=adapter, _chat_id=source.chat_id) -> None:
  520. _adapter.pause_typing_for_chat(_chat_id)
  521. # Non-editing platforms (QQ, WeChat) skip streaming — the partial first message could never
  522. # be updated — unless they have a native-streaming transport (WeCom msgtype "stream").
  523. _adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True)
  524. _adapter_supports_native_stream = bool(getattr(adapter, "SUPPORTS_NATIVE_STREAMING", False))
  525. if not _adapter_supports_edit and not _adapter_supports_native_stream and on_missing_cursor == "raise":
  526. raise RuntimeError("skip streaming for non-editable platform")
  527. _effective_cursor = scfg.cursor if _adapter_supports_edit else ""
  528. # Some Matrix clients render the cursor as tofu: stream text, no cursor.
  529. if source.platform == Platform.MATRIX:
  530. _effective_cursor = ""
  531. # Fresh-final applies to Telegram only (others edit in place cheaply).
  532. # Fresh-final applies to Telegram only — other platforms either edit in place cheaply (Discord,
  533. # Slack) or don't have the timestamp-on-edit / edit-timestamp-stays-stale problem. (Ported from
  534. # openclaw/openclaw#72038.)
  535. _fresh_final_secs = (
  536. float(getattr(scfg, "fresh_final_after_seconds", 0.0) or 0.0)
  537. if source.platform == Platform.TELEGRAM else 0.0
  538. )
  539. _consumer_cfg = StreamConsumerConfig(
  540. edit_interval=scfg.edit_interval, buffer_threshold=scfg.buffer_threshold,
  541. cursor=_effective_cursor,
  542. fresh_final_after_seconds=_fresh_final_secs, transport=scfg.transport or "edit",
  543. chat_type=getattr(source, "chat_type", "") or "",
  544. )
  545. return _consumer_cfg, _pause_typing_before_finalize
  546. def _run_still_current_fn(self, session_key: Optional[str], run_generation: Optional[int]) -> Callable[[], bool]:
  547. """Predicate: does this run's generation still own ``session_key``? (always True when untracked)."""
  548. def _run_still_current() -> bool:
  549. if run_generation is None or not session_key:
  550. return True
  551. return self._is_session_run_current(session_key, run_generation)
  552. return _run_still_current
  553. @staticmethod
  554. def _proxy_error_result(text: str) -> dict[str, Any]:
  555. return {"final_response": text, "messages": [], "api_calls": 0, "tools": []}
  556. def _proxy_stream_consumer(self, source: SessionSource, event_message_id, _thread_metadata, _run_still_current):
  557. """Platform stream consumer for the proxy path when streaming is enabled, else ``None``."""
  558. from gateway.run import _load_gateway_config, _platform_config_key
  559. _scfg = getattr(getattr(self, "config", None), "streaming", None)
  560. # #60671 — streaming TTS consumer is created on the outer event-loop thread before run_sync
  561. # launches. run_sync only reads it via ``streaming_tts_consumer_holder[0]`` for delta callback
  562. # wiring.
  563. if _scfg is None:
  564. from gateway.config import StreamingConfig
  565. _scfg = StreamingConfig()
  566. # Global master switch first: skips the config.yaml re-read on the default (off) path.
  567. if not _scfg.globally_enabled:
  568. return None
  569. from gateway.display_config import resolve_display_setting
  570. _plat_streaming = resolve_display_setting(_load_gateway_config(), _platform_config_key(source.platform), "streaming")
  571. if not _scfg.enabled_for(_plat_streaming):
  572. return None
  573. try:
  574. from gateway.stream_consumer import GatewayStreamConsumer
  575. _adapter = self._delivery_adapter_for(source)
  576. if not _adapter:
  577. return None
  578. _consumer_cfg, _pause_typing_before_finalize = self._build_stream_consumer_config(
  579. source, _scfg, _adapter, on_missing_cursor="fallback",
  580. )
  581. return GatewayStreamConsumer(
  582. adapter=_adapter, chat_id=source.chat_id, config=_consumer_cfg,
  583. metadata=_thread_metadata, on_before_finalize=_pause_typing_before_finalize,
  584. initial_reply_to_id=event_message_id, run_still_current=_run_still_current,
  585. )
  586. except Exception as _sc_err:
  587. logger.debug("Proxy: could not set up stream consumer: %s", _sc_err)
  588. return None
  589. async def _run_agent_via_proxy(
  590. self, message: str, context_prompt: str, history: list[dict[str, Any]],
  591. source: SessionSource, session_id: str, session_key: str | None = None,
  592. run_generation: Optional[int] = None, event_message_id: Optional[str] = None,
  593. scheduled_heartbeat: bool = False,
  594. ) -> dict[str, Any]:
  595. """Forward the message to a remote Hermes API server instead of running a local AIAgent.
  596. Lets a Docker container handle Matrix E2EE while the agent runs on the host with full
  597. access to local files, memory, skills, and a unified session store."""
  598. from gateway.run import _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS
  599. try:
  600. from aiohttp import ClientSession as _AioClientSession, ClientTimeout
  601. except ImportError:
  602. return self._proxy_error_result(t("gateway.proxy.requires_aiohttp", hint=install_hint("messaging")))
  603. proxy_url = self._get_proxy_url()
  604. if not proxy_url:
  605. return self._proxy_error_result(t("gateway.proxy.url_missing"))
  606. # The proxy key is a per-profile credential read through the installed secret scope:
  607. # ``_run_agent`` binds the source profile's scope (or the launch profile's once the process
  608. # multiplexes), and an unscoped single-profile process reads os.environ inside get_secret.
  609. # Any get_secret() error propagates rather than degrading to the ambient key, which may hold
  610. # another profile's credential.
  611. from agent.secret_scope import get_secret_str
  612. proxy_key = get_secret_str("GATEWAY_PROXY_KEY").strip()
  613. _run_still_current = self._run_still_current_fn(session_key, run_generation)
  614. def _stale_result(what: str) -> dict[str, Any]:
  615. logger.info(
  616. "Discarding stale proxy %s for %s — generation %d is no longer current",
  617. what, session_key or "?", run_generation or 0,
  618. )
  619. return {
  620. "final_response": "", "messages": [], "api_calls": 0, "tools": [],
  621. "history_offset": len(history), "session_id": session_id, "response_previewed": False,
  622. }
  623. # OpenAI chat format. The remote keeps continuity via X-Hermes-Session-Id; send the current
  624. # message plus a compact text-only history for a remote that has none yet.
  625. api_messages: list[dict[str, str]] = [{"role": "system", "content": context_prompt}] if context_prompt else []
  626. api_messages += [
  627. {"role": msg.get("role"), "content": msg.get("content")}
  628. for msg in history if msg.get("role") in {"user", "assistant"} and msg.get("content")
  629. ]
  630. api_messages.append({"role": "user", "content": message})
  631. headers: dict[str, str] = {"Content-Type": "application/json"}
  632. if proxy_key:
  633. headers["Authorization"] = f"Bearer {proxy_key}"
  634. if session_id:
  635. headers["X-Hermes-Session-Id"] = session_id
  636. body = {"model": "hermes-agent", "messages": api_messages, "stream": True}
  637. _thread_metadata: Optional[dict[str, Any]] = self._thread_metadata_for_source(source, event_message_id)
  638. _stream_consumer = (
  639. None if scheduled_heartbeat
  640. else self._proxy_stream_consumer(source, event_message_id, _thread_metadata, _run_still_current)
  641. )
  642. stream_task = asyncio.create_task(_stream_consumer.run()) if _stream_consumer else None
  643. _adapter = self._delivery_adapter_for(source)
  644. if _adapter and not scheduled_heartbeat:
  645. with suppress(Exception):
  646. await _adapter.send_typing(source.chat_id, metadata=typing_metadata_for_turn(
  647. _adapter, source.chat_id, event_message_id, _thread_metadata))
  648. full_response = ""
  649. _start = time.time()
  650. saw_done = False
  651. def _consume_sse_line(line: str) -> bool:
  652. """Parse one SSE line into full_response; True when the terminal ``[DONE]`` was seen.
  653. Malformed frames (bad JSON, ``choices: [null]``, non-dict deltas) are skipped —
  654. one bad chunk must not abort the whole stream."""
  655. nonlocal full_response
  656. line = line.strip()
  657. if not line.startswith("data: "):
  658. return False
  659. data = line[6:]
  660. if data.strip() == "[DONE]":
  661. return True
  662. try:
  663. choices = json.loads(data).get("choices") or []
  664. content = choices[0].get("delta", {}).get("content", "") if choices else ""
  665. except (json.JSONDecodeError, TypeError, AttributeError, IndexError):
  666. return False
  667. if content:
  668. full_response += content
  669. if _stream_consumer:
  670. _stream_consumer.on_delta(content)
  671. return False
  672. try:
  673. # sock_connect bounds the TCP connect phase so an unreachable proxy host
  674. # (DNS fail, firewall, remote down) fails fast instead of hanging on the OS default.
  675. _timeout = ClientTimeout(total=0, sock_read=1800, sock_connect=30)
  676. async with _AioClientSession(timeout=_timeout) as session:
  677. async with session.post(f"{proxy_url}/v1/chat/completions", json=body, headers=headers) as resp:
  678. if resp.status != 200:
  679. error_text = await resp.text()
  680. logger.warning("Proxy error (%d) from %s: %s", resp.status, proxy_url, error_text[:500])
  681. return self._proxy_error_result(t("gateway.proxy.http_error", status=resp.status, error=error_text[:300]))
  682. buffer = ""
  683. async for chunk in resp.content.iter_any():
  684. if not _run_still_current():
  685. return _stale_result("stream")
  686. buffer += chunk.decode("utf-8", errors="replace")
  687. while "\n" in buffer:
  688. line, buffer = buffer.split("\n", 1)
  689. if _consume_sse_line(line):
  690. saw_done = True
  691. break
  692. # Exit before the iterator requests another network chunk.
  693. if saw_done:
  694. break
  695. if len(buffer) > _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS:
  696. raise ValueError("Proxy SSE stream exceeded max buffer size without a line boundary")
  697. # The final SSE frame may not be newline-terminated: flush the residual
  698. # buffer after EOF instead of silently dropping its content.
  699. if not saw_done and buffer:
  700. saw_done = _consume_sse_line(buffer)
  701. if not saw_done:
  702. # Clean EOF without [DONE] — the upstream dropped the response
  703. # mid-stream. Keep any partial text but say so instead of
  704. # presenting the truncation as a complete answer.
  705. logger.warning(
  706. "Proxy SSE stream from %s ended without [DONE] — response may be truncated "
  707. "(%d chars received)", proxy_url, len(full_response),
  708. )
  709. if not full_response:
  710. return self._proxy_error_result(t("gateway.proxy.closed_early"))
  711. except asyncio.CancelledError:
  712. raise
  713. except Exception as e:
  714. logger.error("Proxy connection error to %s: %s", proxy_url, e)
  715. if not full_response:
  716. return self._proxy_error_result(t("gateway.proxy.connection_error", error=e))
  717. # Partial response — return what we got
  718. finally:
  719. if _stream_consumer:
  720. _stream_consumer.finish()
  721. if stream_task:
  722. try:
  723. await asyncio.wait_for(stream_task, timeout=5.0)
  724. except (TimeoutError, asyncio.CancelledError):
  725. stream_task.cancel()
  726. _elapsed = time.time() - _start
  727. if not _run_still_current():
  728. return _stale_result("result")
  729. logger.info(
  730. "proxy response: url=%s session=%s time=%.1fs response=%d chars",
  731. proxy_url, (session_id or "")[:20], _elapsed, len(full_response),
  732. )
  733. return {
  734. "final_response": full_response or t("gateway.proxy.no_response"),
  735. "messages": [
  736. {"role": "user", "content": message},
  737. {"role": "assistant", "content": full_response},
  738. ],
  739. "api_calls": 1,
  740. "tools": [],
  741. "history_offset": len(history),
  742. "session_id": session_id,
  743. "response_previewed": _stream_consumer is not None and bool(full_response),
  744. }
  745. async def _run_agent(
  746. self, message: str, context_prompt: str, history: list[dict[str, Any]],
  747. source: SessionSource, session_id: str, **turn_kwargs,
  748. ) -> dict[str, Any]:
  749. """Profile-scoping wrapper around ``_run_agent_inner`` (same keyword parameters; pass-through
  750. when multiplexing is off)."""
  751. with self._profile_scope_for_source(source):
  752. return await self._run_agent_inner(message, context_prompt, history, source, session_id, **turn_kwargs)
  753. def _run_agent_display_settings(self, source: SessionSource) -> GatewayRunner._RunAgentDisplay:
  754. """Resolve per-platform display, progress, status and streaming-surface settings for a turn."""
  755. from gateway.run import (
  756. _gateway_platform_value, _has_platform_display_override, _load_gateway_config,
  757. _platform_config_key,
  758. )
  759. from agent.secret_scope import get_secret
  760. from gateway.display_config import resolve_display_setting, resolve_tool_progress
  761. from gateway.status_phrases import choose_status_phrase, resolve_status_phrase_catalog
  762. from gateway.session_policy import policy_for_source, policy_turn_toolsets
  763. policy = policy_for_source(self, source)
  764. from gateway.session_authorities import active_authority
  765. user_config = policy.config(active_authority(self), secrets=False) if policy else _load_gateway_config()
  766. platform_key = policy.platform if policy else _platform_config_key(source.platform)
  767. enabled_toolsets, disabled_toolsets = self._resolve_turn_toolsets(user_config, source, platform_key)
  768. if policy:
  769. enabled_toolsets = policy_turn_toolsets(active_authority(self), policy, user_config, source.chat_id)
  770. adapter = self._delivery_adapter_for(source)
  771. # Tool preview length (0 = no limit) and friendly tool labels (default on), per-platform.
  772. for _setter, _setting, _default, _cast in (
  773. ("set_tool_preview_max_len", "tool_preview_length", 0, lambda v: int(v) if v else 0),
  774. ("set_friendly_tool_labels", "friendly_tool_labels", True, bool),
  775. ):
  776. with suppress(Exception):
  777. from agent import display as _agent_display
  778. _val = resolve_display_setting(user_config, platform_key, _setting, _default)
  779. getattr(_agent_display, _setter)(_cast(_val))
  780. # Resolve the mode and its provenance together: null inherits, tier off is not intent.
  781. # A raw os.getenv here reads whichever profile's env loaded last under multiplexing
  782. # (#116898); get_secret resolves through the active profile's scope instead.
  783. progress_mode, _tool_progress_explicit = resolve_tool_progress(
  784. user_config, platform_key, get_secret("HERMES_TOOL_PROGRESS_MODE"),
  785. )
  786. # "accumulate" (edit one bubble) or "separate" (one msg per tool)
  787. progress_grouping = resolve_display_setting(user_config, platform_key, "tool_progress_grouping") or "accumulate"
  788. _generic_status_recent: list[str] = []
  789. _generic_status_catalog = resolve_status_phrase_catalog(user_config, platform_key)
  790. def _display_surface_mode(
  791. setting: str, *, default: bool = False,
  792. require_platform_override_for: set[Any] | None = None, allow_generic: bool = False,
  793. ) -> str:
  794. """Return off|raw|generic for a gateway visibility surface."""
  795. if require_platform_override_for:
  796. current_platform = _gateway_platform_value(source.platform)
  797. platform_only = {_gateway_platform_value(item) for item in require_platform_override_for}
  798. if (
  799. current_platform in platform_only
  800. and not _has_platform_display_override(user_config, platform_key, setting)
  801. ):
  802. return "off"
  803. value = resolve_display_setting(user_config, platform_key, setting, default)
  804. if isinstance(value, str) and value.strip().lower() == "generic":
  805. return "generic" if allow_generic else "off"
  806. return "raw" if bool(value) else "off"
  807. def _generic_status_phrase(kind: str, *, tool_name: str | None = None, preview: str | None = None, args: Any = None) -> str:
  808. try:
  809. return choose_status_phrase(
  810. kind, tool_name=tool_name, preview=preview, args=args,
  811. recent=_generic_status_recent, catalog=_generic_status_catalog,
  812. )
  813. except Exception as _phrase_err:
  814. logger.debug("generic status phrase selection failed: %s", _phrase_err)
  815. return (t("gateway.progress.status_fallback_long")
  816. if kind in {"heartbeat", "waiting", "long_running", "status"}
  817. else t("gateway.progress.status_fallback_short"))
  818. # Webhooks can't edit messages; a bot peer gets the final reply only (#137027): no progress lanes.
  819. is_webhook, bot_peer = source.platform == Platform.WEBHOOK, not automatic_notices_allowed(source)
  820. tool_progress_enabled = progress_mode not in {"off", "log"} and not is_webhook and not bot_peer
  821. # Live status for text-rendering typing indicators (Slack); independent of tool_progress.
  822. _live_status_mode = resolve_display_setting(user_config, platform_key, "live_status", "full")
  823. _live_status_adapter = (
  824. adapter if getattr(adapter, "supports_status_text", False) and _live_status_mode != "off" else None
  825. )
  826. # "log" mode: tool calls go to ~/.hermes/logs/tool_calls.log instead of the chat. Gateway-only.
  827. log_mode_enabled = progress_mode == "log" and not is_webhook
  828. # Interim assistant messages and thinking_progress are independent of tool progress (same
  829. # queue). Mattermost requires a per-platform opt-in: scratch text leaks into public threads.
  830. interim_assistant_messages_mode = _display_surface_mode(
  831. "interim_assistant_messages", default=True, require_platform_override_for={Platform.MATTERMOST},
  832. )
  833. interim_assistant_messages_enabled = not is_webhook and not bot_peer and interim_assistant_messages_mode != "off"
  834. _thinking_enabled = _display_surface_mode(
  835. "thinking_progress", default=False, require_platform_override_for={Platform.MATTERMOST},
  836. ) != "off" and not bot_peer
  837. # Slack-native task cards (#29483): when the Slack adapter's opt-in is set, tool progress renders as
  838. # native plan/task cards via chat.startStream — the progress queue is needed even though Slack keeps
  839. # ordinary text tool_progress off by default (requiring both flags would silently leave the native
  840. # feature inactive).
  841. # Cards are still tool progress. Slack's TIER default (``off``) only quiets the text lane so
  842. # cards stay on for unconfigured installs, but an operator who WRITES ``tool_progress: off``
  843. # (global, platform override, or legacy overrides) has asked for no tool progress at all and
  844. # gets no cards either. Every other explicit mode keeps the card lane.
  845. _native_slack_task_cards = False
  846. if (
  847. source.platform == Platform.SLACK
  848. and hasattr(adapter, "native_task_cards_enabled")
  849. and not (_tool_progress_explicit and progress_mode == "off") and not bot_peer
  850. ):
  851. try:
  852. _native_slack_task_cards = bool(adapter.native_task_cards_enabled())
  853. except Exception:
  854. logger.debug("Slack native task-card config check failed", exc_info=True)
  855. return self._RunAgentDisplay(
  856. user_config=user_config, platform_key=platform_key, enabled_toolsets=enabled_toolsets,
  857. disabled_toolsets=disabled_toolsets, resolve_display_setting=resolve_display_setting,
  858. progress_mode=progress_mode, progress_grouping=progress_grouping,
  859. _display_surface_mode=_display_surface_mode,
  860. tool_progress_enabled=tool_progress_enabled, _live_status_mode=_live_status_mode,
  861. _live_status_adapter=_live_status_adapter, log_mode_enabled=log_mode_enabled,
  862. log_queue=queue.Queue() if log_mode_enabled else None,
  863. interim_assistant_messages_enabled=interim_assistant_messages_enabled,
  864. _thinking_enabled=_thinking_enabled, _native_slack_task_cards=_native_slack_task_cards,
  865. needs_progress_queue=tool_progress_enabled or _thinking_enabled or _native_slack_task_cards,
  866. _generic_status_phrase=_generic_status_phrase,
  867. )
  868. # _RunAgentDisplay fields copied verbatim onto the TurnContext.
  869. _DISPLAY_TO_TURN_CTX = (
  870. "_live_status_adapter", "_live_status_mode", "_thinking_enabled", "progress_mode",
  871. "progress_grouping", "tool_progress_enabled", "log_queue", "resolve_display_setting",
  872. "user_config", "enabled_toolsets", "disabled_toolsets", "log_mode_enabled",
  873. "interim_assistant_messages_enabled", "needs_progress_queue", "_native_slack_task_cards",
  874. )
  875. def _run_agent_build_turn_context(
  876. self, disp: GatewayRunner._RunAgentDisplay, AIAgent: Any, *, message: str, source: SessionSource,
  877. session_key: Optional[str], run_generation: Optional[int], **turn_params,
  878. ) -> tuple[TurnContext, TurnRunner, Any]:
  879. """Build the ``TurnContext`` and its ``TurnRunner``; ``turn_params`` (history, context_prompt,
  880. session_id, persist_user_*, …) are stored verbatim. Returns ``(turn_ctx, turn_runner,
  881. cleanup_adapter)``."""
  882. from gateway.run_turn_runner import TurnRunner
  883. # Discord voice "verbal ack" on the FIRST tool call (discord.voice_fx.enabled): resolve the
  884. # guild whose voice connection is bound to this text channel (mirrors DiscordAdapter.play_tts).
  885. _voice_ack_guild: list[Optional[int]] = [self._voice_ack_guild_for(source)]
  886. # Auto-cleanup of temporary progress bubbles needs a real ``delete_message`` (getattr on the
  887. # type: a fake adapter without it means "can't delete", not a crash).
  888. _cleanup_progress = bool(
  889. disp.resolve_display_setting(disp.user_config, disp.platform_key, "cleanup_progress")
  890. )
  891. _cleanup_adapter = self._delivery_adapter_for(source) if _cleanup_progress else None
  892. if _cleanup_adapter is not None and getattr(type(_cleanup_adapter), "delete_message", None) in (
  893. None, BasePlatformAdapter.delete_message,
  894. ):
  895. _cleanup_progress = False
  896. _cleanup_adapter = None
  897. # The one-slot progress/holder containers shared with the callbacks are TurnContext defaults.
  898. turn_ctx = TurnContext(
  899. source=source, message=message, AIAgent=AIAgent, session_key=session_key,
  900. run_generation=run_generation, _cleanup_progress=_cleanup_progress,
  901. _run_still_current=self._run_still_current_fn(session_key, run_generation),
  902. progress_queue=queue.Queue() if disp.needs_progress_queue else None,
  903. _voice_ack_guild=_voice_ack_guild, _voice_ack_loop=asyncio.get_running_loop(),
  904. **{name: getattr(disp, name) for name in self._DISPLAY_TO_TURN_CTX}, **turn_params,
  905. )
  906. turn_runner = TurnRunner(self, turn_ctx)
  907. turn_ctx.mute_notification_reply = diagnostic_turn_muted(
  908. turn_ctx.persist_user_display_metadata, source.platform, turn_ctx.user_config)
  909. # Agent tool-lifecycle callbacks live on the runner (bound methods, same signatures).
  910. turn_ctx.progress_callback = turn_runner.progress_callback
  911. turn_ctx.voice_ack_callback = turn_runner.voice_ack_callback
  912. turn_ctx.native_tool_start_callback = turn_runner.combined_tool_start_callback
  913. turn_ctx.native_tool_complete_callback = turn_runner.native_tool_complete_callback
  914. return turn_ctx, turn_runner, _cleanup_adapter
  915. def _thread_metadata_for_progress(
  916. self, source: SessionSource, event_message_id: Optional[str], _progress_thread_id: Any,
  917. _relay_prospective_thread_id: Optional[str],
  918. ) -> Optional[dict[str, Any]]:
  919. """Thread metadata for a progress-lane send; relay Discord auto-thread lane falls back to the reply anchor.
  920. The connector will auto-thread on the reply anchor (thread is born on its FIRST send), so
  921. carrying it routes progress / status bubbles into the same thread as the final reply."""
  922. if not _progress_thread_id:
  923. metadata = None
  924. elif _progress_thread_id == source.thread_id:
  925. metadata = self._thread_metadata_for_source(source, event_message_id)
  926. else:
  927. metadata = self._thread_metadata_for_target(
  928. source.platform, source.chat_id, _progress_thread_id,
  929. chat_type=getattr(source, "chat_type", None), reply_to_message_id=event_message_id,
  930. )
  931. if metadata is None and _relay_prospective_thread_id:
  932. metadata = {"reply_to_message_id": event_message_id}
  933. return metadata
  934. def _run_agent_progress_threading(
  935. self, source: SessionSource, event_message_id: Optional[str], _native_slack_task_cards: bool
  936. ) -> tuple[Optional[dict], Optional[str], Optional[dict]]:
  937. """Resolve where progress bubbles are threaded (platform-specific).
  938. Returns ``(progress_metadata, progress_reply_to, status_thread_metadata)``; the latter is
  939. for status / approval / stream sends (Feishu topics need the triggering message id via the
  940. reply API, so carry it as a fallback). Slack and Buzz honour the user's reply_in_thread
  941. opt-out: never synthesise a thread for progress, or every later reply inherits it."""
  942. from gateway.run import _non_conversational_metadata, _resolve_progress_thread_id
  943. is_buzz = str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz"
  944. _progress_reply_in_thread = True
  945. _adapter = self._delivery_adapter_for(source) if source.platform == Platform.SLACK or is_buzz else None
  946. if _adapter is not None:
  947. try:
  948. if is_buzz:
  949. _progress_reply_in_thread = getattr(_adapter, "_reply_to_mode", "first") != "off"
  950. else:
  951. # Relay lane: the adapter owns mode resolution; native lane: flat extra key.
  952. _mode_fn = getattr(_adapter, "_effective_reply_in_thread", None)
  953. _progress_reply_in_thread = bool(
  954. _mode_fn() if callable(_mode_fn) else _adapter.config.extra.get("reply_in_thread", True)
  955. )
  956. except Exception:
  957. _progress_reply_in_thread = True
  958. _progress_thread_id = _resolve_progress_thread_id(
  959. source.platform, source.thread_id, event_message_id, reply_in_thread=_progress_reply_in_thread,
  960. )
  961. # Relay Discord auto-thread lane: the connector stamps prospective_thread_id at ingest.
  962. _relay_prospective_thread_id = (
  963. str(getattr(source, "prospective_thread_id", None))
  964. if source.platform == Platform.DISCORD
  965. and getattr(source, "delivered_via_upstream_relay", False)
  966. and getattr(source, "prospective_thread_id", None)
  967. and not source.thread_id
  968. else None
  969. )
  970. _progress_metadata = _non_conversational_metadata(
  971. self._thread_metadata_for_progress(
  972. source, event_message_id, _progress_thread_id, _relay_prospective_thread_id,
  973. ),
  974. platform=source.platform,
  975. )
  976. if _native_slack_task_cards:
  977. # chat.startStream in channels requires the recipient team/user pair; harmless elsewhere.
  978. _progress_metadata = dict(_progress_metadata or {})
  979. if source.scope_id:
  980. _progress_metadata.setdefault("recipient_team_id", source.scope_id)
  981. _progress_metadata.setdefault("slack_team_id", source.scope_id)
  982. if source.user_id:
  983. _progress_metadata.setdefault("recipient_user_id", source.user_id)
  984. # Buzz has no native thread_id: thread via reply-to unless the user opted out.
  985. _progress_reply_to = (
  986. event_message_id
  987. if (source.platform in (Platform.FEISHU, Platform.MATTERMOST) and source.thread_id and event_message_id)
  988. or (is_buzz and event_message_id and _progress_reply_in_thread)
  989. or _relay_prospective_thread_id
  990. else None
  991. )
  992. if source.platform == Platform.FEISHU and source.thread_id and event_message_id:
  993. _status_thread_metadata = {"thread_id": _progress_thread_id, "reply_to_message_id": event_message_id}
  994. else:
  995. _status_thread_metadata = self._thread_metadata_for_progress(
  996. source, event_message_id, _progress_thread_id, _relay_prospective_thread_id,
  997. )
  998. return _progress_metadata, _progress_reply_to, _status_thread_metadata
  999. async def _run_agent_write_tool_log(self, log_queue: Any) -> None:
  1000. """Drain log_queue and append tool-call lines to tool_calls.log (tool_progress=log).
  1001. RotatingFileHandler (5MB × 3) bounds the log; RedactingFormatter keeps secrets off disk."""
  1002. if log_queue is None:
  1003. return
  1004. tool_logger = _tool_call_logger()
  1005. try:
  1006. while True:
  1007. try:
  1008. tool_logger.info("%s", log_queue.get_nowait())
  1009. except queue.Empty:
  1010. await asyncio.sleep(0.3)
  1011. except Exception as e:
  1012. logger.error("write_tool_log error: %s", e)
  1013. await asyncio.sleep(1)
  1014. except asyncio.CancelledError:
  1015. pass
  1016. finally:
  1017. # Drain remaining entries so late tool calls from the final iteration aren't lost.
  1018. with suppress(Exception):
  1019. while True:
  1020. tool_logger.info("%s", log_queue.get_nowait())
  1021. with suppress(Exception):
  1022. for handler in tool_logger.handlers:
  1023. handler.flush()
  1024. def _run_agent_start_streaming_tts(
  1025. self, source: SessionSource, message_type: Optional[str],
  1026. _status_thread_metadata: Optional[dict[str, Any]], streaming_tts_consumer_holder: list,
  1027. ) -> None:
  1028. """Start the streaming-TTS consumer for a voice-input turn on an auto-TTS chat.
  1029. Created on the gateway loop thread (not run_sync's executor); an inactive consumer leaves
  1030. the holder None so the whole-file fallback path runs."""
  1031. # Skip when streaming TTS already delivered audio for this turn (#60671).
  1032. # This avoids a cross-scope NameError: the outer interrupt / finalisation paths reference the
  1033. # consumer via ``streaming_tts_consumer_holder[0]``. Gates: voice input, auto-TTS enabled for this
  1034. # chat, adapter supports streaming, and a usable streaming TTS provider configured. See #60671.
  1035. _stts_adapter = self._delivery_adapter_for(source)
  1036. _is_voice_input = (
  1037. message_type is not None
  1038. and str(getattr(message_type, "value", message_type)).lower() == "voice"
  1039. )
  1040. if _stts_adapter is None or not _is_voice_input or not _stts_adapter._should_auto_tts_for_chat(source.chat_id):
  1041. return
  1042. try:
  1043. from gateway.streaming_tts_consumer import StreamingTTSConsumer
  1044. from tools.tts_tool import _load_tts_config
  1045. _stts_consumer = StreamingTTSConsumer(
  1046. adapter=_stts_adapter, chat_id=source.chat_id, tts_config=_load_tts_config(),
  1047. loop=self._gateway_loop or asyncio.get_event_loop(),
  1048. metadata=_status_thread_metadata,
  1049. )
  1050. if _stts_consumer.active:
  1051. streaming_tts_consumer_holder[0] = _stts_consumer
  1052. _stts_consumer.start()
  1053. except Exception as _stts_err:
  1054. logger.debug("Could not set up streaming TTS consumer: %s", _stts_err)
  1055. async def _run_agent_stream_consumer_task(self, stream_consumer_holder: list) -> None:
  1056. """Wait (up to 10s) for the stream consumer to be created inside run_sync, then run it."""
  1057. for _ in range(200):
  1058. if stream_consumer_holder[0] is not None:
  1059. await stream_consumer_holder[0].run()
  1060. return
  1061. await asyncio.sleep(0.05)
  1062. @staticmethod
  1063. async def _await_stream_task(stream_task) -> None:
  1064. """Give the stream consumer task 5s to flush, then cancel it."""
  1065. try:
  1066. await asyncio.wait_for(stream_task, timeout=5.0)
  1067. except (TimeoutError, asyncio.CancelledError):
  1068. stream_task.cancel()
  1069. with suppress(asyncio.CancelledError):
  1070. await stream_task
  1071. async def _run_agent_track_agent(self, turn_ctx: TurnContext) -> None:
  1072. """Track this agent as running for the session (interrupt support) once it is created — only
  1073. if this run is still current, else leave the newer run's slot alone."""
  1074. session_key, run_generation, agent_holder = turn_ctx.session_key, turn_ctx.run_generation, turn_ctx.agent_holder
  1075. while agent_holder[0] is None:
  1076. await asyncio.sleep(0.05)
  1077. if not session_key:
  1078. return
  1079. if run_generation is not None and not self._is_session_run_current(session_key, run_generation):
  1080. logger.info(
  1081. "Skipping stale agent promotion for %s — generation %s is no longer current",
  1082. session_key or "", run_generation,
  1083. )
  1084. return
  1085. turn_state = self._session_state(session_key).turn
  1086. turn_state.agent, turn_state.ctx = agent_holder[0], turn_ctx
  1087. if self._draining:
  1088. self._update_runtime_status("draining")
  1089. async def _run_agent_fire_pending_interrupt(
  1090. self, adapter: Any, agent: Any, source: SessionSource, session_key: str,
  1091. _interrupt_detected: asyncio.Event, streaming_tts_consumer_holder: list, *,
  1092. log_context: str, log: Callable[[], None],
  1093. ) -> None:
  1094. """Peek the adapter's pending event, transcribe voice, then signal the agent + abort streaming TTS.
  1095. Peek WITHOUT consuming: the event must stay for the post-run ``_dequeue_pending_event()``
  1096. (popping races the agent finishing). Transcribe BEFORE signaling so voice interrupts carry
  1097. the real transcript."""
  1098. from gateway.run import _build_media_placeholder
  1099. _peek_event = adapter._pending_messages.get(session_key)
  1100. pending_text = None
  1101. if _peek_event is not None:
  1102. pending_text = _peek_event.text or ""
  1103. if self._pending_event_audio_paths(_peek_event):
  1104. pending_text, _ = await self._transcribe_and_echo_pending_voice(
  1105. _peek_event, adapter, source, pending_text, log_context=log_context,
  1106. metadata={"thread_id": source.thread_id} if source.thread_id else None,
  1107. )
  1108. elif not pending_text and (getattr(_peek_event, "media_urls", None) or []):
  1109. pending_text = _build_media_placeholder(_peek_event)
  1110. log()
  1111. agent.interrupt(pending_text)
  1112. _interrupt_detected.set()
  1113. # Abort streaming TTS on barge-in.
  1114. # See #60671.
  1115. # See #60671.
  1116. # See #60671.
  1117. # Finalize the streaming-TTS consumer (#60671). finish() is called from the outer event-loop thread
  1118. # (not the executor worker) so early returns from run_sync are also finalised. wait_complete()
  1119. # drains queued audio; on timeout the consumer is aborted unconditionally — if audio was audible,
  1120. # suppression is preserved so the gateway does not replay from the beginning; if no audio was
  1121. # audible, the whole-file fallback path is permitted.
  1122. _stts = streaming_tts_consumer_holder[0]
  1123. if _stts is not None:
  1124. _stts.abort("barge-in")
  1125. async def _run_agent_monitor_for_interrupt(self, turn_ctx: TurnContext, _interrupt_detected: asyncio.Event) -> None:
  1126. """Poll the adapter for interrupts (new messages) every 200ms and signal the agent.
  1127. Level 1 (base.py) catches regular text before _handle_message(); the inactivity poll loop
  1128. has a BACKUP check in case this task dies. Keyed by session_key, NOT source.chat_id."""
  1129. source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder
  1130. streaming_tts_consumer_holder = turn_ctx.streaming_tts_consumer_holder
  1131. if not session_key:
  1132. return
  1133. while True:
  1134. await asyncio.sleep(0.2)
  1135. try:
  1136. # Re-resolve the adapter each iteration so reconnects don't leave a stale reference.
  1137. _adapter = self._delivery_adapter_for(source)
  1138. if not _adapter:
  1139. continue
  1140. if hasattr(_adapter, 'has_pending_interrupt') and _adapter.has_pending_interrupt(session_key):
  1141. agent = agent_holder[0]
  1142. if agent:
  1143. await self._run_agent_fire_pending_interrupt(
  1144. _adapter, agent, source, session_key, _interrupt_detected,
  1145. streaming_tts_consumer_holder, log_context="Voice-interrupt",
  1146. log=lambda: logger.debug("Interrupt detected from adapter, signaling agent..."),
  1147. )
  1148. break
  1149. except asyncio.CancelledError:
  1150. raise
  1151. except Exception as _mon_err:
  1152. logger.debug("monitor_for_interrupt error (will retry): %s", _mon_err)
  1153. async def _run_agent_backup_interrupt_check(
  1154. self, turn_ctx: TurnContext, _interrupt_detected: asyncio.Event, interrupt_monitor: asyncio.Task,
  1155. ) -> None:
  1156. """Backup interrupt check: if the monitor task died or missed the interrupt, catch it here."""
  1157. source, session_key = turn_ctx.source, turn_ctx.session_key
  1158. if _interrupt_detected.is_set() or not session_key:
  1159. return
  1160. _backup_adapter = self._delivery_adapter_for(source)
  1161. _backup_agent = turn_ctx.agent_holder[0]
  1162. if (_backup_adapter and _backup_agent
  1163. and hasattr(_backup_adapter, 'has_pending_interrupt')
  1164. and _backup_adapter.has_pending_interrupt(session_key)):
  1165. await self._run_agent_fire_pending_interrupt(
  1166. _backup_adapter, _backup_agent, source, session_key, _interrupt_detected,
  1167. turn_ctx.streaming_tts_consumer_holder,
  1168. log_context="Voice-backup-interrupt",
  1169. log=lambda: logger.info(
  1170. "Backup interrupt detected for session %s (monitor task state: %s)",
  1171. session_key, "done" if interrupt_monitor.done() else "running",
  1172. ),
  1173. )
  1174. @staticmethod
  1175. def _run_agent_stream_confirmed_final_delivery(consumer, final_text: str, *, previewed: bool = False) -> bool:
  1176. """True only when the actual final reply reached the user: a finalize call may carry only the
  1177. last preview snapshot, so reconcile against the recorded payload — a demonstrable mismatch
  1178. (False) overrides the flag; None keeps legacy trust."""
  1179. if consumer is None:
  1180. return False
  1181. if getattr(consumer, "final_response_sent", False):
  1182. matcher = getattr(consumer, "delivered_final_matches", None)
  1183. if callable(matcher):
  1184. with suppress(Exception):
  1185. if matcher(final_text) is False:
  1186. return False
  1187. return True
  1188. # Exact-text match against what the consumer DURABLY delivered (commentary, segments, and the
  1189. # visible prefix only once a real send landed) — safe without the ``previewed`` flag. The codex
  1190. # app-server bridge delivers the final agentMessage through the commentary path and never sets
  1191. # response_previewed (#74248 / #80519); gating on the flag re-sent every such reply. Mismatching
  1192. # commentary still returns False, so a distinct final answer is never suppressed (#65919). Draft
  1193. # frames are ephemeral and must not count: after draft streaming + a failed finalize send this
  1194. # predicate must stay False so the fallback final send still fires (#51828 / #33793).
  1195. has_delivered_text = getattr(consumer, "has_durably_delivered_text", None)
  1196. if callable(has_delivered_text):
  1197. try:
  1198. return bool(has_delivered_text(final_text))
  1199. except Exception:
  1200. return False
  1201. return False
  1202. def _run_agent_start_turn_worker(self, turn_ctx: TurnContext, run_sync: Callable[[], Any]) -> GatewayRunner._RunAgentWorker:
  1203. """Schedule ``run_sync`` on the executor plus the inactivity watchdog thread.
  1204. *Inactivity* timeout (agent.gateway_timeout / HERMES_AGENT_TIMEOUT, env wins; 0 = unlimited),
  1205. not wall-clock. The daemon watchdog is independent of asyncio: cgroup memory reclaim can
  1206. starve the loop that runs the normal timeout poll."""
  1207. from gateway.run import _float_env, _watch_gateway_turn_inactivity
  1208. from tools.process_registry import process_registry
  1209. agent_holder, session_key, run_generation = turn_ctx.agent_holder, turn_ctx.session_key, turn_ctx.run_generation
  1210. _agent_timeout, _agent_warning = (
  1211. v if v > 0 else None
  1212. for v in (_float_env("HERMES_AGENT_TIMEOUT", 1800), _float_env("HERMES_AGENT_TIMEOUT_WARNING", 900))
  1213. )
  1214. # background=true processes survive a turn: reap only children created by THIS turn on timeout.
  1215. _turn_task_id = turn_ctx.session_id or ""
  1216. # The daemon watchdog is independent of asyncio: cgroup memory reclaim may starve the event loop
  1217. # that runs the normal timeout poll, but it need not also postpone cleanup until the loop recovers
  1218. # (#76115).
  1219. _turn_process_baseline = process_registry.snapshot_running_ids(_turn_task_id)
  1220. turn_ctx.process_task_id = _turn_task_id
  1221. turn_ctx.process_baseline = _turn_process_baseline
  1222. # task_id is session-scoped: gate the reap on this claim still being current so a replacement
  1223. # turn's fresh process isn't killed by this turn's stale baseline.
  1224. worker = self._RunAgentWorker(
  1225. agent_timeout=_agent_timeout, agent_warning=_agent_warning, task_id=_turn_task_id,
  1226. process_baseline=_turn_process_baseline, worker_done=threading.Event(),
  1227. timeout_fired=threading.Event(), cleanup_lock=threading.Lock(),
  1228. is_current=(
  1229. (lambda: self._is_session_run_current(session_key, run_generation))
  1230. if run_generation is not None
  1231. else (lambda: True)
  1232. ),
  1233. )
  1234. def _run_sync_with_timeout_lifecycle():
  1235. try:
  1236. return run_sync()
  1237. finally:
  1238. worker.worker_done.set()
  1239. # `.turn.agent` stays reachable until the *next* turn is claimed; clearing the
  1240. # ownership markers now means a /stop on the finished turn no longer reaps background
  1241. # work it left running.
  1242. # `.turn.agent` on the session state is only reset to _AGENT_PENDING_SENTINEL when the
  1243. # *next* turn is claimed (see _session_state(...).turn.agent = ... at claim time), so a
  1244. # stale reference to this exact agent instance stays reachable from
  1245. # _interrupt_and_clear_session() until then. See #76115.
  1246. _finished_agent = agent_holder[0] if agent_holder else None
  1247. if _finished_agent is not None:
  1248. _finished_agent._gateway_turn_process_task_id = ""
  1249. _finished_agent._gateway_turn_process_baseline = frozenset()
  1250. if _agent_timeout is not None:
  1251. threading.Thread(
  1252. target=_watch_gateway_turn_inactivity,
  1253. kwargs={
  1254. "agent_holder": agent_holder, "timeout": _agent_timeout, "poll_interval": 5.0,
  1255. **self._reaper_kwargs(worker),
  1256. },
  1257. name=f"gateway-turn-watchdog-{_turn_task_id[:12]}",
  1258. daemon=True,
  1259. ).start()
  1260. from gateway.session_runtime_workers import start_turn_worker
  1261. start_turn_worker(self, worker, agent_holder, _run_sync_with_timeout_lifecycle)
  1262. return worker
  1263. @staticmethod
  1264. def _reaper_kwargs(worker: GatewayRunner._RunAgentWorker) -> dict:
  1265. """Shared kwargs of the watchdog + timeout-reaper threads."""
  1266. return {
  1267. **{k: getattr(worker, k) for k in ("task_id", "process_baseline", "worker_done", "timeout_fired", "cleanup_lock")},
  1268. "is_still_current": worker.is_current,
  1269. }
  1270. @staticmethod
  1271. def _agent_activity_summary(agent: Any) -> dict:
  1272. """``agent.get_activity_summary()`` or ``{}`` when unavailable / failing."""
  1273. if agent and hasattr(agent, "get_activity_summary"):
  1274. with suppress(Exception):
  1275. return agent.get_activity_summary()
  1276. return {}
  1277. async def _run_agent_inactivity_warning(self, worker, source, _status_thread_metadata) -> None:
  1278. """Staged one-shot warning before the inactivity timeout escalates."""
  1279. from gateway.run import _interim_metadata
  1280. _warn_adapter = self._delivery_adapter_for(source) if automatic_notices_allowed(source) else None
  1281. if not _warn_adapter:
  1282. return
  1283. try:
  1284. await _warn_adapter.emit_warning(
  1285. source.chat_id, t(
  1286. "gateway.progress.stuck_warning",
  1287. minutes=int(worker.agent_warning // 60) or 1,
  1288. remaining=int((worker.agent_timeout - worker.agent_warning) // 60) or 1),
  1289. metadata=_interim_metadata(_status_thread_metadata), logical_platform=source.platform,
  1290. )
  1291. except Exception as _warn_err:
  1292. logger.debug("Inactivity warning send error: %s", _warn_err)
  1293. def _run_agent_timeout_result(self, worker, turn_ctx: TurnContext) -> dict:
  1294. """Synthetic failed run dict for an inactivity timeout, with the activity-tracker diagnostic;
  1295. interrupts the agent if it is still running so the thread pool worker is freed."""
  1296. from gateway.run import _INTERRUPT_REASON_TIMEOUT, _INTERRUPT_TOOL_REASON_TIMEOUT, request_hard_interrupt
  1297. from gateway.session_results import record_terminal_failure
  1298. session_key, result_holder, tools_holder = turn_ctx.session_key, turn_ctx.result_holder, turn_ctx.tools_holder
  1299. _timed_out_agent = turn_ctx.agent_holder[0]
  1300. _activity = self._agent_activity_summary(_timed_out_agent)
  1301. _last_desc = _activity.get("last_activity_desc", "unknown")
  1302. _secs_ago = _activity.get("seconds_since_activity", 0)
  1303. _cur_tool = _activity.get("current_tool")
  1304. _iter_n = _activity.get("api_call_count", 0)
  1305. _iter_max = _activity.get("max_iterations", 0)
  1306. # Operator-facing log keeps the raw resolved value; only the user-facing lines hide the sentinel.
  1307. logger.error(
  1308. "Agent idle for %.0fs (timeout %.0fs) in session %s "
  1309. "| last_activity=%s | iteration=%s/%s | tool=%s",
  1310. _secs_ago, worker.agent_timeout, session_key, _last_desc, _iter_n, _iter_max,
  1311. _cur_tool or "none",
  1312. )
  1313. if _timed_out_agent:
  1314. request_hard_interrupt(_timed_out_agent, _INTERRUPT_REASON_TIMEOUT, tool_reason=_INTERRUPT_TOOL_REASON_TIMEOUT)
  1315. _timeout_mins = int(worker.agent_timeout // 60) or 1
  1316. _iter_progress = format_iteration_progress(_iter_n, _iter_max)
  1317. _diag_lines = [t("gateway.progress.timeout_header", minutes=_timeout_mins)]
  1318. if _cur_tool:
  1319. _diag_lines.append(t(
  1320. "gateway.progress.timeout_stuck_tool",
  1321. tool=_cur_tool, seconds=f"{_secs_ago:.0f}", progress=_iter_progress))
  1322. else:
  1323. _diag_lines.append(t(
  1324. "gateway.progress.timeout_last_activity",
  1325. activity=_last_desc, seconds=f"{_secs_ago:.0f}", progress=_iter_progress))
  1326. _diag_lines.append(t("gateway.progress.timeout_hint"))
  1327. return record_terminal_failure({
  1328. "final_response": "\n".join(_diag_lines),
  1329. "messages": result_holder[0].get("messages", []) if result_holder[0] else [],
  1330. "api_calls": _iter_n,
  1331. "tools": tools_holder[0] or [],
  1332. "history_offset": 0,
  1333. "failed": True,
  1334. })
  1335. async def _run_agent_await_turn_worker(
  1336. self, worker: GatewayRunner._RunAgentWorker, turn_ctx: TurnContext,
  1337. _interrupt_detected: asyncio.Event, interrupt_monitor: asyncio.Task,
  1338. ) -> Any:
  1339. """Poll the executor future (inactivity timeout + backup interrupt checks); return its result,
  1340. or a synthetic failed run dict on inactivity timeout. Polls even with an unlimited timeout
  1341. so the backup interrupt check runs if monitor_for_interrupt() silently died."""
  1342. from gateway.run import _abandon_timed_out_gateway_turn
  1343. agent_holder = turn_ctx.agent_holder
  1344. _warning_fired = False
  1345. while True:
  1346. done, _ = await asyncio.wait({worker.executor_task}, timeout=5.0)
  1347. if done:
  1348. # Prefer the real result even if the watchdog fired in the same window (the run already
  1349. # persisted its reply).
  1350. return worker.executor_task.result()
  1351. if worker.agent_timeout is not None:
  1352. if worker.timeout_fired.is_set():
  1353. break
  1354. _idle_secs = turn_ctx.awake_idle.measure(self._agent_activity_summary(agent_holder[0]).get("seconds_since_activity", 0.0))
  1355. if not _warning_fired and worker.agent_warning is not None and _idle_secs >= worker.agent_warning:
  1356. _warning_fired = True
  1357. await self._run_agent_inactivity_warning(worker, turn_ctx.source, turn_ctx._status_thread_metadata)
  1358. if _idle_secs >= worker.agent_timeout:
  1359. threading.Thread(
  1360. target=_abandon_timed_out_gateway_turn,
  1361. kwargs={"agent_holder": agent_holder, **self._reaper_kwargs(worker)},
  1362. name=f"gateway-turn-reaper-{worker.task_id[:12]}", daemon=True,
  1363. ).start()
  1364. break
  1365. await self._run_agent_backup_interrupt_check(turn_ctx, _interrupt_detected, interrupt_monitor)
  1366. return self._run_agent_timeout_result(worker, turn_ctx)
  1367. def _run_agent_evict_on_fallback(self, turn_ctx: TurnContext) -> None:
  1368. """Evict the cached agent when a fallback model activated on a SUCCESSFUL run (so /model shows
  1369. the active model and the next message retries the primary). Skip failed runs: evicting
  1370. would loop bad model → fallback → evict → recreate."""
  1371. session_key, _agent, _result_for_fb = turn_ctx.session_key, turn_ctx.agent_holder[0], turn_ctx.result_holder[0]
  1372. if _agent is None or not hasattr(_agent, 'model') or (_result_for_fb and _result_for_fb.get("failed")):
  1373. return
  1374. # A provider fallback is drift even when it serves the configured model name on another endpoint.
  1375. if getattr(_agent, "_provider_fallback_active", False) is not True:
  1376. # A LOCAL session runs its frozen launch policy's model, not the gateway's configured one.
  1377. from gateway.session_authorities import active_authority
  1378. from gateway.session_policy import policy_for_source
  1379. policy = policy_for_source(self, turn_ctx.source)
  1380. _cfg_model = self._fallback_baseline_model(
  1381. session_key, turn_ctx.source, _agent, policy_model=policy.current_model(active_authority(self)) if policy and policy.model else None)
  1382. if _agent.model == _cfg_model or self._is_intentional_model_switch(session_key, _agent, _cfg_model):
  1383. return
  1384. _agent._evicted_on_fallback = session_key or True
  1385. self._evict_cached_agent(session_key)
  1386. async def _run_agent_finalize_streaming_tts(self, turn_ctx: TurnContext, adapter: Any) -> None:
  1387. """Finalize the streaming-TTS consumer on the outer event-loop thread (covers early returns
  1388. from run_sync). On drain timeout abort to free the task — audible streams keep whole-file
  1389. suppression, silent streams stay eligible for the whole-file fallback."""
  1390. _stts = turn_ctx.streaming_tts_consumer_holder[0]
  1391. if _stts is None:
  1392. return
  1393. _stts.finish()
  1394. try:
  1395. await _stts.wait_complete(timeout=10.0)
  1396. except Exception as _stts_done_err:
  1397. logger.debug("streaming TTS wait_complete error: %s", _stts_done_err)
  1398. if not _stts.done:
  1399. _stts.abort("streaming TTS finalisation timeout")
  1400. await _stts.wait_complete(timeout=2.0)
  1401. if _stts.suppress_whole_file and adapter is not None:
  1402. _mark_turn = getattr(adapter, "_mark_streaming_tts_completed_turn", None)
  1403. if callable(_mark_turn):
  1404. _mark_turn(turn_ctx.session_key, turn_ctx.run_generation)
  1405. async def _run_agent_drain_pending(
  1406. self, result: Any, adapter: Any, source: SessionSource, session_key: Optional[str]
  1407. ) -> tuple[Any, Optional[str]]:
  1408. """Dequeue the adapter's pending / interrupt / leftover-steer follow-up as ``(pending_event, pending)``.
  1409. Keyed by session_key (not source.chat_id) to match the adapter's storage keys."""
  1410. from gateway.run import _build_media_placeholder, _dequeue_pending_event, _is_control_interrupt_message
  1411. from gateway.session_ingress import executing_admission
  1412. pending_event = None
  1413. pending = None
  1414. if result and adapter and session_key:
  1415. pending_event = _dequeue_pending_event(adapter, session_key)
  1416. if result.get("compression_exhausted") and pending_event is not None:
  1417. self._defer_past_exhaustion_reset(result, adapter, source, session_key, pending_event)
  1418. return None, None
  1419. # /queue overflow: promote the next queued event into the consumed "next-up" slot so the
  1420. # recursive drain sees it (keeps FIFO order; a mid-chain /queue can't jump the queue).
  1421. pending_event = self._promote_queued_event(session_key, adapter, pending_event)
  1422. if result.get("interrupted") and not pending_event and result.get("interrupt_message"):
  1423. interrupt_message = result.get("interrupt_message")
  1424. if _is_control_interrupt_message(interrupt_message):
  1425. logger.info(
  1426. "Ignoring control interrupt message for session %s: %s",
  1427. session_key or "?", interrupt_message,
  1428. )
  1429. else:
  1430. # FIFO already has this send.
  1431. pending = None if executing_admission.get() else interrupt_message
  1432. elif pending_event:
  1433. # Transcribe audio BEFORE it becomes the next user turn (real transcript, not a path).
  1434. _pending_text = pending_event.text or ""
  1435. if self._pending_event_audio_paths(pending_event):
  1436. pending, _ = await self._transcribe_and_echo_pending_voice(
  1437. pending_event, adapter, source, _pending_text, log_context="Voice-drain",
  1438. metadata={"thread_id": source.thread_id} if source.thread_id else None,
  1439. )
  1440. pending = pending or _build_media_placeholder(pending_event)
  1441. else:
  1442. pending = _pending_text or _build_media_placeholder(pending_event)
  1443. if pending:
  1444. logger.debug("Processing queued message after agent completion: '%s...'", pending[:40])
  1445. # Leftover /steer (arrived after the last tool batch): deliver as the next user turn.
  1446. pending_event, pending = self._merge_leftover_steer(result, adapter, source, session_key, pending_event, pending)
  1447. # Safety net: a pending slash command is never passed to the agent as user input.
  1448. if pending and pending.strip().startswith("/"):
  1449. _pending_cmd_word = pending.strip().split(None, 1)[0][1:].lower()
  1450. if _pending_cmd_word:
  1451. with suppress(Exception):
  1452. from hermes_cli.commands import resolve_command as _rc_pending
  1453. if _rc_pending(_pending_cmd_word):
  1454. logger.info(
  1455. "Discarding command '/%s' from pending queue — "
  1456. "commands must not be passed as agent input", _pending_cmd_word,
  1457. )
  1458. pending_event = None
  1459. pending = None
  1460. if self._draining and getattr(pending_event, "_admitted_before_drain", False) is not True:
  1461. if pending_event or pending:
  1462. logger.info("Discarding pending follow-up for session %s during gateway %s", session_key or "?", self._status_action_label())
  1463. # Without a result the caller runs nothing next: leave the queue to the adapter.
  1464. pending_event = self._take_wake_admitted_before_drain(session_key, adapter) if result and adapter and session_key else None
  1465. pending = (pending_event.text or _build_media_placeholder(pending_event)) if pending_event is not None else None
  1466. return pending_event, pending
  1467. async def _run_agent_deliver_first_response(
  1468. self, turn_ctx: TurnContext, adapter: Any, response: Any, result: Any, stream_task: Any,
  1469. ) -> None:
  1470. """Deliver the first response before a queued follow-up runs, unless streaming already did."""
  1471. if turn_ctx.mute_notification_reply:
  1472. return
  1473. session_key = turn_ctx.session_key
  1474. _sc = turn_ctx.stream_consumer_holder[0]
  1475. if _sc and stream_task:
  1476. try:
  1477. await self._await_stream_task(stream_task)
  1478. except Exception as e:
  1479. logger.debug("Stream consumer wait before queued message failed: %s", e)
  1480. # Delivery uses the finalized task result (empty/failure normalization), not raw ``result``.
  1481. _delivery_result = response if isinstance(response, dict) else (result or {})
  1482. first_response = _delivery_result.get("final_response", "")
  1483. _already_streamed = self._run_agent_stream_confirmed_final_delivery(
  1484. _sc, first_response, previewed=bool(_delivery_result.get("response_previewed")),
  1485. )
  1486. # Same silence predicate as the normal path, else this branch leaks the literal marker.
  1487. _silent = False
  1488. if self._is_intentional_silence(_delivery_result, first_response):
  1489. _silent = self._silence_allowed_for_source(
  1490. turn_ctx.source, turn_ctx.persist_user_display_kind, turn_ctx.reply_expected, _delivery_result)
  1491. if _silent:
  1492. logger.info(
  1493. "Queued follow-up for session %s: suppressing intentional silence marker before continuing.",
  1494. session_key or "?",
  1495. )
  1496. first_response = ""
  1497. else:
  1498. logger.warning(
  1499. "Queued follow-up for session %s: replacing a human-turn silence marker.",
  1500. session_key or "?",
  1501. )
  1502. first_response = _unexpected_silence_reply()
  1503. _already_streamed = False
  1504. await self._record_room_silence_decision(
  1505. session_key, _delivery_result, _silent,
  1506. display_kind=turn_ctx.persist_user_display_kind, reply_expected=turn_ctx.reply_expected)
  1507. # Failed turns deliver their text but never their attachments (completed-turn parity).
  1508. _deliver_media = not _delivery_result.get("failed")
  1509. if first_response:
  1510. logger.info(
  1511. "Queued follow-up for session %s: final text delivery confirmed; delivering explicit media before continuing."
  1512. if _already_streamed else
  1513. "Queued follow-up for session %s: final stream delivery not confirmed; sending first response before continuing.",
  1514. session_key or "?",
  1515. )
  1516. try:
  1517. _text_delivered = await self._deliver_queued_first_response(
  1518. first_response, source=turn_ctx.source, adapter=adapter,
  1519. metadata=turn_ctx._status_thread_metadata, event_message_id=turn_ctx.event_message_id,
  1520. text_already_delivered=_already_streamed,
  1521. deliver_media=_deliver_media, stream_consumer=_sc,
  1522. # The text send records a delivery-ledger obligation under this key, keyed on
  1523. # the raw inbound id (the anchor above is only the reply target).
  1524. session_key=session_key, inbound_message_id=turn_ctx.inbound_message_id,
  1525. )
  1526. except Exception as e:
  1527. logger.warning("Failed to send first response before queued message: %s", e)
  1528. else:
  1529. # One source of truth for "this turn's final already reached the chat": the normal
  1530. # completion path (`_hmwa_deliver_turn_response`) consults ``already_sent`` on the
  1531. # result the queued lane hands back. Every early `return result` after this point
  1532. # (follow-up text refused, stale goal continuation) otherwise re-sends the text the
  1533. # fallback just delivered — the #81052 duplicate. A REFUSED send reports False, and
  1534. # the completion send stays the fallback so the user is not left with nothing.
  1535. if _text_delivered and isinstance(result, dict):
  1536. result["already_sent"] = True
  1537. # The queued lane already uploaded this response's MEDIA: attachments; without
  1538. # this the completion path's already_sent rescan uploads every file twice.
  1539. result["media_already_delivered"] = _deliver_media
  1540. # Release deferred bg-review notifications: pop (no double-fire in base.py's finally) and call.
  1541. _bg_cb = self._pop_post_delivery_callback(adapter, session_key, turn_ctx.run_generation)
  1542. if callable(_bg_cb):
  1543. with suppress(Exception):
  1544. _bg_result = _bg_cb()
  1545. if inspect.isawaitable(_bg_result):
  1546. await _bg_result
  1547. async def _run_agent_queued_followup(
  1548. self, turn_ctx: TurnContext, adapter: Any, pending: Optional[str], pending_event: Any,
  1549. response: Any, result: Any, stream_task: Any,
  1550. ) -> Any:
  1551. """Run the queued / interrupting follow-up as the next turn (recursive ``_run_agent``)."""
  1552. from gateway.run import _preserve_queued_followup_history_offset
  1553. source, session_id, session_key, run_generation = (
  1554. turn_ctx.source, turn_ctx.session_id, turn_ctx.session_key, turn_ctx.run_generation,
  1555. )
  1556. _interrupt_depth, history, _status_thread_metadata = (
  1557. turn_ctx._interrupt_depth, turn_ctx.history, turn_ctx._status_thread_metadata,
  1558. )
  1559. logger.debug("Processing pending message: '%s...'", pending[:40])
  1560. # Clear the interrupt event so the recursive _run_agent isn't re-interrupted (infinite loop).
  1561. _active = getattr(adapter, "_active_sessions", None) if adapter else None
  1562. if _active and session_key and session_key in _active:
  1563. _active[session_key].clear()
  1564. # Cap recursion depth (user keeps sending while the agent keeps failing).
  1565. # (#816)
  1566. if _interrupt_depth >= self._MAX_INTERRUPT_DEPTH:
  1567. logger.warning(
  1568. "Interrupt recursion depth %d reached for session %s — "
  1569. "queueing message instead of recursing.", _interrupt_depth, session_key,
  1570. )
  1571. self._requeue_at_depth_cap(session_key, source, pending_event, pending)
  1572. return turn_ctx.result_holder[0] or {"final_response": response, "messages": history}
  1573. # Interrupted: discard the response ("Operation interrupted." is noise).
  1574. if not result.get("interrupted"):
  1575. await self._run_agent_deliver_first_response(turn_ctx, adapter, response, result, stream_task)
  1576. updated_history = result.get("messages", history)
  1577. next_source, next_message, next_session_key = source, pending, session_key
  1578. # message_type is carried into the recursive call so queued voice turns can stream TTS.
  1579. next_message_id = next_channel_prompt = next_message_type = None
  1580. # The raw inbound id keys the delivery-ledger obligation for the follow-up's own final send,
  1581. # distinct from the reply anchor above (None in forum topics). Carry it or two chained
  1582. # topic turns with the same text would collide on one obligation id (queued-final-ledger).
  1583. next_inbound_id = None
  1584. # Queued Discord turns carry the same routing note as first turns; persist the authored text.
  1585. next_persist_message = None
  1586. next_display_kind = display_kind_for_event(pending_event)
  1587. next_reply_expected = pending_event.reply_expected if pending_event is not None else None
  1588. # See #60671.
  1589. if pending_event is not None:
  1590. next_source = getattr(pending_event, "source", None) or source
  1591. if self._is_goal_continuation_event(pending_event) and not self._goal_still_active_for_session(session_id):
  1592. logger.info(
  1593. "Discarding stale goal continuation for session %s — goal is no longer active",
  1594. session_key or "?",
  1595. )
  1596. return result
  1597. # Resolve the follow-up's session key BEFORE preparing the inbound text: native image
  1598. # paths are buffered under the key given and consumed under next_session_key.
  1599. try:
  1600. next_session_key = self._session_key_for_source(next_source)
  1601. except Exception:
  1602. logger.debug(
  1603. "Queued follow-up session-key resolution failed; reusing %s",
  1604. session_key or "?", exc_info=True,
  1605. )
  1606. next_message = await self._prepare_profile_scoped_inbound_message_text(
  1607. event=pending_event, source=next_source, history=updated_history, session_key=next_session_key,
  1608. )
  1609. if next_message is None:
  1610. return result
  1611. from gateway.run_inbound import strip_discord_triggering_note
  1612. next_persist_message = strip_discord_triggering_note(pending_event, next_message)
  1613. next_message_id = self._reply_anchor_for_event(pending_event)
  1614. next_inbound_id = str(pending_event.message_id) if getattr(pending_event, "message_id", None) else None
  1615. next_channel_prompt, next_source = self._pinned_channel_inputs(
  1616. next_session_key, pending_event.channel_prompt, next_source,
  1617. internal=pending_event.internal or getattr(pending_event, "inherit_channel_pin", False),
  1618. )
  1619. if not pending_event.internal:
  1620. # A drained human turn re-pins its channel inputs; make them durable like a first turn.
  1621. await self._persist_prompt_pins(next_session_key, session_id)
  1622. next_message_type = getattr(pending_event, "message_type", None)
  1623. else:
  1624. # Event-less interrupt/steer follow-ups continue the effective prompt
  1625. # of the turn they are recursively following.
  1626. next_channel_prompt = turn_ctx.channel_prompt
  1627. # Clear the prior turn's streaming-TTS completion marker so the recursive turn isn't suppressed (#60671).
  1628. _clear_adapter = self._delivery_adapter_for(source)
  1629. _completed_turns = getattr(_clear_adapter, "_streaming_tts_completed_turns", None)
  1630. _prior_key = getattr(_clear_adapter, "_streaming_tts_turn_key", None)
  1631. if _completed_turns is not None and callable(_prior_key) and session_key and run_generation is not None:
  1632. _pk = _prior_key(session_key, run_generation)
  1633. if _pk:
  1634. _completed_turns.discard(_pk)
  1635. # Restart the typing indicator; the outer typing task may be stale.
  1636. if _clear_adapter:
  1637. with suppress(Exception):
  1638. await _clear_adapter.send_typing(source.chat_id, metadata=typing_metadata_for_turn(
  1639. _clear_adapter, source.chat_id, next_message_id, _status_thread_metadata))
  1640. # Re-baseline the cached agent's message_count snapshot before recursing into the in-band queued
  1641. # (/queue) follow-up turn. The first turn has completed and flushed its own user + assistant rows to
  1642. # the SessionDB, so the cross-process coherence guard (#45966) — which this recursive _run_agent
  1643. # call re-enters — would otherwise see the grown on-disk count against the stale build-time snapshot
  1644. # and rebuild the agent on THIS process's OWN writes, destroying the prompt-cache prefix #46237 was
  1645. # merged to preserve. The existing re-baseline in _handle_message_with_agent only runs after the
  1646. # whole _run_agent chain unwinds — too late for the in-band follow-up. Use the same (session_key,
  1647. # session_id) the recursive call runs under so the snapshot matches exactly what the follow-up's
  1648. # guard will consult. Fail-safe in helper.
  1649. # Acknowledge the follow-up the way an idle-session message is: this in-band drain is the only
  1650. # place a queued/interrupting message ever runs, so base.py's hook site is never entered for it.
  1651. # Resolve the adapter from the follow-up's OWN source — a multiplexed gateway can route it to a
  1652. # different profile's adapter, and only that instance holds the per-message reaction state.
  1653. from gateway.run_turn_followup_ack import _followup_cancel_outcome, _run_followup_processing_hook
  1654. _hook_adapter = self._intake_adapter_for(next_source) if pending_event is not None else None
  1655. await _run_followup_processing_hook(_hook_adapter, pending_event, "on_processing_start")
  1656. # The re-baseline sits inside the try: a /stop landing on its DB await must still close the marker
  1657. # (the helper's own ``except Exception`` does not catch cancellation).
  1658. try:
  1659. await self._refresh_agent_cache_message_count(session_key, session_id)
  1660. followup_result = await self._run_agent(
  1661. message=next_message, context_prompt=turn_ctx.context_prompt, history=updated_history,
  1662. source=next_source, session_id=session_id, session_key=next_session_key,
  1663. run_generation=run_generation, _interrupt_depth=_interrupt_depth + 1,
  1664. event_message_id=next_message_id, inbound_message_id=next_inbound_id,
  1665. channel_prompt=next_channel_prompt, message_type=next_message_type,
  1666. persist_user_message=next_persist_message,
  1667. persist_user_display_kind=next_display_kind,
  1668. reply_expected=next_reply_expected,
  1669. persist_user_display_metadata={
  1670. **display_metadata_for_event(pending_event),
  1671. **reply_expected_metadata(next_reply_expected), **diagnostic_metadata(pending_event)} or None,
  1672. )
  1673. except asyncio.CancelledError:
  1674. await _run_followup_processing_hook(
  1675. _hook_adapter, pending_event, "on_processing_complete", _followup_cancel_outcome(_hook_adapter))
  1676. raise
  1677. except BaseException:
  1678. await _run_followup_processing_hook(
  1679. _hook_adapter, pending_event, "on_processing_complete", ProcessingOutcome.FAILURE)
  1680. raise
  1681. await _run_followup_processing_hook(
  1682. _hook_adapter, pending_event, "on_processing_complete", ProcessingOutcome.SUCCESS)
  1683. merged = _preserve_queued_followup_history_offset(result, followup_result)
  1684. # The TERMINAL turn of the chain owns the ledger identity for the outer final send, which
  1685. # the adapter brackets against the event that OPENED the chain. Without this the terminal
  1686. # reply is recorded under the first message's id, so a first reply that was refused (flood
  1687. # control) has its outstanding row replaced and marked delivered by an identical-text
  1688. # terminal reply, and is never redelivered. A deeper recursion has already set its own id,
  1689. # so only fill the key while it is still absent: the innermost turn wins.
  1690. if isinstance(merged, dict) and "queued_terminal_inbound_id" not in merged:
  1691. merged = {
  1692. **merged,
  1693. "queued_terminal_inbound_id": next_inbound_id,
  1694. "queued_terminal_display_kind": next_display_kind,
  1695. "queued_terminal_reply_expected": next_reply_expected,
  1696. "queued_terminal_source": next_source,
  1697. "queued_terminal_notification_category": (
  1698. (pending_event.metadata or {}).get("notification_category", "result")
  1699. if pending_event is not None and pending_event.internal else "result"),
  1700. }
  1701. return merged
  1702. async def _run_agent_cleanup_turn_tasks(
  1703. self, turn_ctx: TurnContext, *, progress_task: Any, log_task: Any, interrupt_monitor: asyncio.Task,
  1704. _notify_task: asyncio.Task, tracking_task: asyncio.Task, stream_task: Any,
  1705. ) -> None:
  1706. """``finally`` half of a turn: cancel background tasks, flush stream, release the session slot."""
  1707. stream_consumer_holder, session_key = turn_ctx.stream_consumer_holder, turn_ctx.session_key
  1708. for task in (progress_task, log_task, interrupt_monitor, _notify_task):
  1709. if task:
  1710. task.cancel()
  1711. if stream_task:
  1712. # No stream consumer was created: nothing to flush, cancel instead of waiting out 5s.
  1713. if not (stream_consumer_holder and stream_consumer_holder[0] is not None):
  1714. stream_task.cancel()
  1715. with suppress(asyncio.CancelledError):
  1716. await stream_task
  1717. else:
  1718. await self._await_stream_task(stream_task)
  1719. # Abort + bounded wait for streaming TTS: covers paths where normal finalisation was skipped.
  1720. _stts_finally = turn_ctx.streaming_tts_consumer_holder[0]
  1721. # See #60671.
  1722. if _stts_finally is not None and not _stts_finally.done:
  1723. _stts_finally.abort("cleanup")
  1724. with suppress(Exception):
  1725. await _stts_finally.wait_complete(timeout=2.0)
  1726. tracking_task.cancel()
  1727. if session_key:
  1728. # Release the slot only if this run's generation still owns it (/stop or /new may have
  1729. # installed its own state).
  1730. self._release_running_agent_state(session_key, run_generation=turn_ctx.run_generation)
  1731. if self._draining:
  1732. self._update_runtime_status("draining")
  1733. for task in (progress_task, log_task, interrupt_monitor, tracking_task, _notify_task):
  1734. if task:
  1735. try:
  1736. await task
  1737. except asyncio.CancelledError:
  1738. pass
  1739. except Exception:
  1740. # A background task that died of a real error must not abort the cleanup path.
  1741. logger.debug("background turn task failed during cleanup", exc_info=True)
  1742. async def _run_agent_edit_streamed_message(
  1743. self, _sc, source, response, content, *, _sk, ok, fail_result: str, fail_exc: str,
  1744. ) -> None:
  1745. """Edit the stream consumer's message in place with ``content``; on success mark
  1746. ``response["already_sent"]`` and log ``ok``. A returned failure logs ``fail_result`` as
  1747. ``(session, error)`` and an exception logs ``fail_exc`` as ``(session, exc)``; either way
  1748. ``already_sent`` stays unset so the normal final send delivers the content."""
  1749. try:
  1750. _res = await _sc.adapter.edit_message(
  1751. chat_id=source.chat_id, message_id=_sc.message_id, content=content, finalize=True,
  1752. )
  1753. except Exception as _edit_err:
  1754. logger.warning(fail_exc, _sk, _edit_err)
  1755. return
  1756. if not getattr(_res, "success", True):
  1757. logger.warning(fail_result, _sk, getattr(_res, "error", None))
  1758. return
  1759. response["already_sent"] = True
  1760. logger.info(*ok)
  1761. async def _run_agent_mark_streamed_delivery(self, response: Any, turn_ctx: TurnContext) -> None:
  1762. """Set ``response["already_sent"]`` when streaming already delivered the final reply.
  1763. Never when the agent failed (unseen error), on "(empty)", or on a finite (`-z`, `chat -q`) turn:
  1764. its viewer prints only the completion text, never the deltas (#139666). Both flags reflect call
  1765. success, not content: a mismatch with the recorded turn-final payload never suppresses."""
  1766. from gateway.session_finite import finite_turn_required
  1767. _sc, source, session_key = turn_ctx.stream_consumer_holder[0], turn_ctx.source, turn_ctx.session_key
  1768. if not isinstance(response, dict) or response.get("failed") or finite_turn_required() is True:
  1769. return
  1770. _final = response.get("final_response") or ""
  1771. _is_empty_sentinel = not _final or _final == "(empty)"
  1772. # response_previewed: only suppress if that EXACT text was delivered, not unrelated commentary.
  1773. # Unrelated commentary/progress must not be mistaken for the final response (#14238).
  1774. _previewed = bool(response.get("response_previewed"))
  1775. _content_delivered = bool(_sc and getattr(_sc, "final_content_delivered", False))
  1776. # #71643: a *successful* finalize edit can still carry only the last preview snapshot — deltas
  1777. # generated between that edit and stream completion never reach any API call, and both suppression
  1778. # flags are set from the call's success rather than its content. Reconcile the consumer's recorded
  1779. # turn-final payload against the completed response: on a demonstrable mismatch (False) neither
  1780. # final_response_sent nor final_content_delivered may suppress the normal final send. False also
  1781. # covers payload-less multi-message split delivery (#78541). None (no record on a non-split legacy
  1782. # path) keeps legacy trust; the failed-finalize family (#51828 / #33793) is unaffected because those
  1783. # paths leave the flags False or record the complete fallback payload.
  1784. _stale_finalized = False
  1785. if _content_delivered and not _is_empty_sentinel:
  1786. _matcher = getattr(_sc, "delivered_final_matches", None)
  1787. if callable(_matcher):
  1788. with suppress(Exception):
  1789. _stale_finalized = _matcher(_final) is False
  1790. if _stale_finalized:
  1791. _content_delivered = False
  1792. # Plugin hooks may append content after streaming finished — then send the final version.
  1793. _transformed = bool(response.get("response_transformed"))
  1794. # Suppress the normal send only when the actual final reply reached the user.
  1795. _streamed = self._run_agent_stream_confirmed_final_delivery(_sc, _final, previewed=_previewed)
  1796. if _is_empty_sentinel:
  1797. return
  1798. _sk = session_key or "?"
  1799. if not _transformed and (_streamed or _content_delivered):
  1800. logger.info(
  1801. "Suppressing normal final send for session %s: final delivery already confirmed (streamed=%s previewed=%s content_delivered=%s).",
  1802. _sk, _streamed, _previewed, _content_delivered,
  1803. )
  1804. response["already_sent"] = True
  1805. elif not _transformed and _stale_finalized and _sc is not None:
  1806. # Stale finalize: edit the streamed message up to the complete response (on failure the
  1807. # normal send delivers). Not for split delivery — message_id is only the LAST chunk.
  1808. _sc_msg_id = _sc.message_id
  1809. if getattr(_sc, "_turn_split_delivery", False):
  1810. logger.info(
  1811. "Stale streamed finalize detected for session %s on a multi-message split; skipping the in-place reconciliation edit and delivering the complete response via normal final send (#78541).",
  1812. _sk,
  1813. )
  1814. elif _sc_msg_id and _sc_msg_id != "__no_edit__" and getattr(_sc, "adapter", None) is not None:
  1815. await self._run_agent_edit_streamed_message(
  1816. _sc, source, response, _final, _sk=_sk,
  1817. ok=("Reconciled stale streamed finalize for session %s: edited message %s with the complete response (#71643).", _sk, _sc_msg_id),
  1818. fail_result="Stale-finalize reconciliation edit failed for session %s (%s); sending complete response via normal final send.",
  1819. fail_exc="Stale-finalize reconciliation edit failed for session %s: %s; sending complete response via normal final send.",
  1820. )
  1821. else:
  1822. logger.info(
  1823. "Stale streamed finalize detected for session %s with no editable message; delivering complete response via normal final send (#71643).",
  1824. _sk,
  1825. )
  1826. elif _transformed and _sc is not None:
  1827. # Transformed after streaming: edit the streamed message instead of sending a duplicate.
  1828. if _sc.message_id:
  1829. await self._run_agent_edit_streamed_message(
  1830. _sc, source, response, response["final_response"], _sk=_sk,
  1831. ok=("Edited streamed message %s for session %s to include plugin-transformed content.", _sc.message_id, _sk),
  1832. fail_result="Transformed-final edit failed for session %s (%s); sending transformed response via normal final send.",
  1833. fail_exc="Failed to edit streamed message for session %s: %s",
  1834. )
  1835. elif _sc is not None and getattr(_sc, "stream_deltas_enabled", True):
  1836. # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed but suppression did NOT fire; log
  1837. # the decision inputs ("signal never set" vs "ack-pending race"). Skipped for consumers
  1838. # never fed the final's deltas (interim-only wiring, #105341) — they cannot have raced
  1839. # the normal final send, so the warning would be a guaranteed false positive.
  1840. logger.warning(
  1841. "Normal final-send NOT suppressed despite active stream consumer for session %s: "
  1842. "streamed=%s previewed=%s content_delivered=%s transformed=%s final_len=%d — "
  1843. "possible duplicate send (see wecom ack-timeout RCA).",
  1844. _sk, _streamed, _previewed, _content_delivered, _transformed, len(_final),
  1845. )
  1846. def _run_agent_schedule_bubble_cleanup(self, response: Any, _cleanup_adapter: Any, turn_ctx: TurnContext) -> None:
  1847. """Schedule deletion of tracked temporary progress bubbles after the final response lands.
  1848. Failed runs keep them as breadcrumbs. Only on adapters with ``delete_message``; failures swallowed."""
  1849. from gateway.run import safe_schedule_threadsafe
  1850. _cleanup_msg_ids, session_key = turn_ctx._cleanup_msg_ids, turn_ctx.session_key
  1851. if not (
  1852. turn_ctx._cleanup_progress
  1853. and _cleanup_adapter is not None
  1854. and _cleanup_msg_ids
  1855. and session_key
  1856. and isinstance(response, dict)
  1857. and not response.get("failed")
  1858. and hasattr(_cleanup_adapter, "register_post_delivery_callback")
  1859. ):
  1860. return
  1861. _ids_snapshot = list(_cleanup_msg_ids)
  1862. _chat_id_snapshot = turn_ctx.source.chat_id
  1863. _loop_snapshot = asyncio.get_running_loop()
  1864. def _cleanup_temp_bubbles() -> None:
  1865. async def _delete_all() -> None:
  1866. for _mid in _ids_snapshot:
  1867. with suppress(Exception):
  1868. await _cleanup_adapter.delete_message(_chat_id_snapshot, _mid)
  1869. with suppress(Exception):
  1870. safe_schedule_threadsafe(
  1871. _delete_all(), _loop_snapshot, logger=logger,
  1872. log_message="Temp bubble cleanup scheduling error",
  1873. )
  1874. try:
  1875. _cleanup_adapter.register_post_delivery_callback(
  1876. session_key, _cleanup_temp_bubbles, generation=turn_ctx.run_generation,
  1877. )
  1878. except Exception as _rpe:
  1879. logger.debug("Post-delivery cleanup registration failed: %s", _rpe)
  1880. def _run_agent_bind_turn_wiring(
  1881. self, turn_ctx: TurnContext, turn_runner: TurnRunner, source: SessionSource,
  1882. event_message_id: Optional[str], _native_slack_task_cards: bool,
  1883. ) -> Optional[dict[str, Any]]:
  1884. """Resolve progress threading, then publish progress metadata and the sync→async bridges onto
  1885. ``turn_ctx`` (the one-slot holders shared with run_sync's executor thread are TurnContext
  1886. defaults). Returns ``_status_thread_metadata``."""
  1887. turn_ctx._progress_metadata, turn_ctx._progress_reply_to, _status_thread_metadata = (
  1888. self._run_agent_progress_threading(source, event_message_id, _native_slack_task_cards)
  1889. )
  1890. # Bridges: sync step/event/status callbacks → async hooks.emit and adapter.send.
  1891. turn_ctx._loop_for_step = asyncio.get_running_loop()
  1892. turn_ctx._hooks_ref = self.hooks
  1893. turn_ctx._step_callback_sync = turn_runner._step_callback_sync
  1894. turn_ctx._event_callback_sync = turn_runner._event_callback_sync
  1895. turn_ctx._status_callback_sync = turn_runner._status_callback_sync
  1896. turn_ctx._status_adapter = self._delivery_adapter_for(source)
  1897. turn_ctx._status_chat_id = source.chat_id
  1898. turn_ctx._status_thread_metadata = _status_thread_metadata
  1899. return _status_thread_metadata
  1900. async def _run_agent_notify_long_running(
  1901. self, disp: GatewayRunner._RunAgentDisplay, turn_ctx: TurnContext, _executor_task_holder: list,
  1902. ) -> None:
  1903. """Periodic "still working" heartbeat, edited in place where supported. Stops once this run
  1904. no longer owns the session slot or the executor finished. ``_executor_task_holder[0]`` is
  1905. bound just after this task is scheduled (reads as None until then).
  1906. Interval: agent.gateway_notify_interval / HERMES_AGENT_NOTIFY_INTERVAL (default 180s; 0 or
  1907. long_running_notifications=off disables)."""
  1908. from gateway.run import _float_env, _interim_metadata, _non_conversational_metadata
  1909. _notify_start = time.time()
  1910. _NOTIFY_INTERVAL = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180)
  1911. _long_running_mode = disp._display_surface_mode("long_running_notifications", default=True, allow_generic=True)
  1912. if _NOTIFY_INTERVAL <= 0 or _long_running_mode == "off":
  1913. return
  1914. source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder
  1915. _status_thread_metadata = turn_ctx._status_thread_metadata
  1916. _notify_adapter = self._delivery_adapter_for(source)
  1917. if not _notify_adapter:
  1918. return
  1919. _heartbeat_msg_id: Optional[str] = None
  1920. while True:
  1921. await asyncio.sleep(_NOTIFY_INTERVAL)
  1922. if not self._should_emit_long_running_notification(
  1923. session_key, agent_holder[0], _executor_task_holder[0]
  1924. ):
  1925. break
  1926. _elapsed_mins = int((time.time() - _notify_start) // 60)
  1927. # Terse heartbeat by default; the iteration counter is gated on busy_ack_detail.
  1928. _status_detail = ""
  1929. _want_iteration_detail = bool(
  1930. disp.resolve_display_setting(disp.user_config, disp.platform_key, "busy_ack_detail", True)
  1931. )
  1932. _a = self._agent_activity_summary(agent_holder[0])
  1933. with suppress(Exception):
  1934. if _a:
  1935. _parts = []
  1936. if _want_iteration_detail:
  1937. _parts.append(format_iteration_progress(_a["api_call_count"], _a["max_iterations"]))
  1938. _action = _a.get("current_tool") or _a.get("last_activity_desc")
  1939. if _action:
  1940. _parts.append(str(_action))
  1941. if _parts:
  1942. _status_detail = " — " + ", ".join(_parts)
  1943. _heartbeat_text = (
  1944. disp._generic_status_phrase("status")
  1945. if _long_running_mode == "generic"
  1946. else t("gateway.progress.working_heartbeat", minutes=_elapsed_mins, detail=_status_detail)
  1947. )
  1948. try:
  1949. _notify_res = None
  1950. if _heartbeat_msg_id:
  1951. try:
  1952. _notify_res = await _notify_adapter.edit_message(source.chat_id, _heartbeat_msg_id, _heartbeat_text)
  1953. except Exception as _ee:
  1954. logger.debug("Heartbeat edit failed: %s", _ee)
  1955. _notify_res = None
  1956. if not (_notify_res and getattr(_notify_res, "success", False)):
  1957. # The edit above awaited; a drain/restart notice may have gone out meanwhile, and
  1958. # a fresh "Working" bubble after it reads as a contradiction (#10990).
  1959. if not self._should_emit_long_running_notification(
  1960. session_key, agent_holder[0], _executor_task_holder[0]
  1961. ):
  1962. break
  1963. _notify_res = await _notify_adapter.send(
  1964. source.chat_id, _heartbeat_text,
  1965. metadata=_interim_metadata(_non_conversational_metadata(_status_thread_metadata, platform=source.platform)),
  1966. )
  1967. if getattr(_notify_res, "success", False) and getattr(_notify_res, "message_id", None):
  1968. _heartbeat_msg_id = str(_notify_res.message_id)
  1969. if turn_ctx._cleanup_progress:
  1970. turn_ctx._cleanup_msg_ids.append(_heartbeat_msg_id)
  1971. except Exception as _ne:
  1972. logger.debug("Long-running notification error: %s", _ne)
  1973. async def _run_agent_inner(
  1974. self, message: str, context_prompt: str, history: list[dict[str, Any]],
  1975. source: SessionSource, session_id: str, session_key: str | None = None,
  1976. run_generation: Optional[int] = None, _interrupt_depth: int = 0,
  1977. event_message_id: Optional[str] = None, inbound_message_id: Optional[str] = None,
  1978. channel_prompt: Optional[str] = None, moa_config: Optional[dict] = None,
  1979. persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None,
  1980. persist_user_display_kind: Optional[str] = None, message_type: Optional[str] = None,
  1981. persist_user_display_metadata: Optional[dict] = None,
  1982. reply_expected: Optional[bool] = None,
  1983. scheduled_heartbeat: bool = False,
  1984. title_user_message: Optional[str] = None,
  1985. ) -> dict[str, Any]:
  1986. """Run the agent; returns the full run_conversation result dict.
  1987. Keys: "final_response", "messages", "api_calls", "completed"."""
  1988. if self._get_proxy_url():
  1989. result = await self._run_agent_via_proxy(
  1990. message=message, context_prompt=context_prompt, history=history, source=source,
  1991. session_id=session_id, session_key=session_key, run_generation=run_generation,
  1992. event_message_id=event_message_id, scheduled_heartbeat=scheduled_heartbeat,
  1993. )
  1994. # The remote turn is this admission's executed turn (``TurnRunner.run_sync`` records
  1995. # the local one): without it a finite run would settle as a no-execution refusal.
  1996. from gateway.session_results import execution_result
  1997. captured = execution_result.get()
  1998. if captured is not None and 'result' not in captured:
  1999. captured['result'] = result
  2000. return result
  2001. from run_agent import AIAgent
  2002. disp = await asyncio.to_thread(self._run_agent_display_settings, source) # auth lock + config I/O; scope rides the copied context
  2003. if scheduled_heartbeat:
  2004. # A heartbeat is proactive work: tool chrome, drafts, thinking and periodic
  2005. # liveness notices would create a user-visible ping before its final result is known.
  2006. # Keep status callbacks intact for approvals and actionable failures. Interim messages keep the chat's
  2007. # setting (TurnRunner drops them for a heartbeat): an agent this turn builds takes its mode from it.
  2008. disp = dataclasses.replace(
  2009. disp, tool_progress_enabled=False, _thinking_enabled=False, _native_slack_task_cards=False,
  2010. needs_progress_queue=False,
  2011. )
  2012. turn_ctx, turn_runner, _cleanup_adapter = self._run_agent_build_turn_context(
  2013. disp, AIAgent, message=message, source=source, session_key=session_key,
  2014. run_generation=run_generation, context_prompt=context_prompt, history=history,
  2015. session_id=session_id, _interrupt_depth=_interrupt_depth,
  2016. event_message_id=event_message_id, inbound_message_id=inbound_message_id,
  2017. channel_prompt=channel_prompt, moa_config=moa_config,
  2018. title_user_message=title_user_message,
  2019. persist_user_message=persist_user_message,
  2020. persist_user_timestamp=persist_user_timestamp,
  2021. persist_user_display_kind=persist_user_display_kind,
  2022. reply_expected=reply_expected,
  2023. persist_user_display_metadata=persist_user_display_metadata, scheduled_heartbeat=scheduled_heartbeat,
  2024. voice_turn=str(getattr(message_type, "value", message_type) or "").lower() == "voice",
  2025. )
  2026. _status_thread_metadata = self._run_agent_bind_turn_wiring(
  2027. turn_ctx, turn_runner, source, event_message_id, disp._native_slack_task_cards,
  2028. )
  2029. # Two independent quiet reasons: a muted diagnostic wake (ours) and a scheduled heartbeat.
  2030. if not (scheduled_heartbeat or turn_ctx.mute_notification_reply):
  2031. self._run_agent_start_streaming_tts(
  2032. source, message_type, _status_thread_metadata, turn_ctx.streaming_tts_consumer_holder,
  2033. )
  2034. # Progress sender drains BOTH tool-progress lines and thinking bubbles (needs_progress_queue).
  2035. spawn = asyncio.create_task
  2036. progress_task = spawn(turn_runner.send_progress_messages()) if disp.needs_progress_queue else None
  2037. log_task = spawn(self._run_agent_write_tool_log(disp.log_queue)) if disp.log_mode_enabled else None
  2038. # The stream consumer is created inside run_sync; this task polls for it.
  2039. stream_task = spawn(self._run_agent_stream_consumer_task(turn_ctx.stream_consumer_holder))
  2040. tracking_task = spawn(self._run_agent_track_agent(turn_ctx))
  2041. _interrupt_detected = asyncio.Event() # shared with backup check
  2042. interrupt_monitor = spawn(self._run_agent_monitor_for_interrupt(turn_ctx, _interrupt_detected))
  2043. # Periodic "still working" notifications so the user knows the agent hasn't died.
  2044. _executor_task_holder: list = [None] # bound once the executor future exists (see below)
  2045. _notify_task = (
  2046. None if (scheduled_heartbeat or turn_ctx.mute_notification_reply or not automatic_notices_allowed(source))
  2047. else spawn(self._run_agent_notify_long_running(disp, turn_ctx, _executor_task_holder))
  2048. )
  2049. try:
  2050. # run_sync is TurnRunner.run_sync (bound method; executor call unchanged).
  2051. worker = self._run_agent_start_turn_worker(turn_ctx, turn_runner.run_sync)
  2052. _executor_task_holder[0] = worker.executor_task # read late by _notify_long_running
  2053. response = await self._run_agent_await_turn_worker(worker, turn_ctx, _interrupt_detected, interrupt_monitor)
  2054. if isinstance(response, dict):
  2055. response["_notification_reply_muted"] = turn_ctx.mute_notification_reply
  2056. self._run_agent_evict_on_fallback(turn_ctx)
  2057. # Interrupted OR queued message (/queue)?
  2058. result = turn_ctx.result_holder[0]
  2059. adapter = self._delivery_adapter_for(source)
  2060. await self._run_agent_finalize_streaming_tts(turn_ctx, adapter)
  2061. pending_event, pending = await self._run_agent_drain_pending(result, adapter, source, session_key)
  2062. if pending_event or pending:
  2063. return await self._run_agent_queued_followup(
  2064. turn_ctx, adapter, pending, pending_event, response, result, stream_task,
  2065. )
  2066. finally:
  2067. await self._run_agent_cleanup_turn_tasks(
  2068. turn_ctx, progress_task=progress_task, log_task=log_task, interrupt_monitor=interrupt_monitor,
  2069. _notify_task=_notify_task, tracking_task=tracking_task, stream_task=stream_task,
  2070. )
  2071. await self._run_agent_mark_streamed_delivery(response, turn_ctx)
  2072. self._run_agent_schedule_bubble_cleanup(response, _cleanup_adapter, turn_ctx)
  2073. return response