| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239 |
- """Agent-turn execution for GatewayRunner: _handle_message_with_agent, _run_agent*, proxy path,
- MCP reload. Bound onto ``GatewayRunner`` via the MRO; ``gateway.run`` internals
- are imported lazily inside method bodies (import cycle) so ``patch("gateway.run.X")`` still works.
- """
- from __future__ import annotations
- from pm import install_hint
- import logging
- from typing import TYPE_CHECKING
- import asyncio
- import dataclasses
- import inspect
- import json
- import os
- import queue
- import threading
- import time
- from agent.i18n import t
- from agent.session_activity import format_iteration_progress
- from agent.turn_failure_copy import FAILED_TURN_DISPLAY_KIND, FAILED_TURN_NOTICE, PARTIAL_FAILED_TURN_NOTICE
- from contextlib import nullcontext, suppress
- from contextvars import copy_context
- from gateway.bot_peer_notices import automatic_notices_allowed
- from gateway.config import Platform
- from gateway.platforms.base import BasePlatformAdapter, ProcessingOutcome
- from gateway.platforms.event import MessageEvent
- from gateway.platforms.helpers import typing_metadata_for_turn
- from gateway.response_filters import (
- display_kind_for_event, display_metadata_for_event, is_machinery_display_kind, persisted_user_metadata_for_event,
- reply_expected_metadata,
- )
- from gateway.warning_notifications import diagnostic_metadata, diagnostic_turn_muted, diagnostic_wake_muted
- from gateway.session import (
- SessionSource, _session_key_namespace, build_channel_continuity_note,
- build_session_context,
- )
- from gateway.session_transcript import TranscriptReadError
- from gateway.turn_context import TurnContext
- from gateway.turn_lease import DEFAULT_LEASE_WAIT, TurnLeaseTimeoutError
- from hermes_constants import get_hermes_home_override
- from pathlib import Path
- from typing import Any, Callable, Dict, List, Optional, Tuple
- from utils import base_url_hostname
- if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
- from gateway.run import GatewayRunner
- from gateway.run_turn_runner import TurnRunner
- # Log-record parity with the origin module.
- logger = logging.getLogger("gateway.run")
- _tool_call_logger_lock = threading.Lock()
- def _tool_call_logger() -> logging.Logger:
- """Process-wide ``hermes.tool_calls`` Logger + one RotatingFileHandler on logs/tool_calls.log.
- Named Loggers live in ``logging.Logger.manager.loggerDict`` forever, so the former per-turn name
- (``hermes.tool_calls.<id(log_queue)>``) leaked one Logger per logged turn (#62950); a single
- shared handler also keeps concurrent turns from double-writing lines."""
- tool_logger = logging.getLogger("hermes.tool_calls")
- with _tool_call_logger_lock:
- if not tool_logger.handlers:
- from logging.handlers import RotatingFileHandler
- from agent.redact import RedactingFormatter
- from gateway.run import _hermes_home
- log_dir = _hermes_home / "logs"
- log_dir.mkdir(parents=True, exist_ok=True)
- handler = RotatingFileHandler(
- log_dir / "tool_calls.log", maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8",
- )
- handler.setFormatter(RedactingFormatter("%(message)s"))
- tool_logger.setLevel(logging.INFO)
- tool_logger.propagate = False
- tool_logger.addHandler(handler)
- return tool_logger
- _CONTEXT_OVERFLOW_ERROR_PHRASES = (
- "context length", "context size", "context window",
- "maximum context", "token limit", "too many tokens",
- "reduce the length", "exceeds the limit",
- "request entity too large", "prompt is too long",
- "payload too large", "input is too long",
- )
- def _unexpected_silence_reply() -> str:
- """Reply when the model returned only a silence marker for a message that needed an answer."""
- return t("gateway.errors.unexpected_silence")
- def is_context_overflow_failure_result(agent_result: dict, history_len: int) -> bool:
- """One verdict for "this failed turn is a context overflow", shared by transcript persistence
- (#1630 skip) and the user-facing reply so the two can never disagree.
- Multi-word phrases (not bare "exceed"/"token") avoid matching "rate limit exceeded" or
- "invalid authentication token"; a bare 400 only counts on a long session."""
- if not agent_result.get("failed"):
- return False
- if agent_result.get("compression_exhausted"):
- return True
- err = str(agent_result.get("error") or "").lower()
- return any(p in err for p in _CONTEXT_OVERFLOW_ERROR_PHRASES) or ("400" in err and history_len > 50)
- # Setup/prefix rows rather than conversation: the agent rebuilds its own system prompt, and a
- # transcript meta row is logging-only — neither reaches the model, but both are the head a
- # fail-closed payload keeps.
- _HYGIENE_SETUP_ROLES = ("system", "session_meta")
- def bound_model_input_without_hygiene(history: list[Any], limit: int) -> list[Any]:
- """Fail-closed in-context bound for a turn where hygiene has not landed (#111988).
- Keeps the leading ``system``/``session_meta`` setup rows plus the newest tail, total <= ``limit``.
- Deterministic (the same transcript always yields the same cut) and payload-only: the stored
- transcript is never touched, so the agent's durable-prefix slice (``history_offset``) is
- unaffected. Returns ``history`` unchanged — same object — when nothing needs dropping, so the
- landed-compression and below-the-limit paths stay byte-identical.
- """
- if len(history) <= limit:
- return history
- head_end = 0
- while (head_end < len(history) and isinstance(history[head_end], dict)
- and history[head_end].get("role") in _HYGIENE_SETUP_ROLES):
- head_end += 1
- # Always leave room for the newest row: a setup-only payload would answer nothing.
- head_end = min(head_end, limit - 1)
- tail_start = len(history) - (limit - head_end)
- # Never start the kept tail on a tool result: its parent assistant(tool_calls) row is dropped
- # with it, and an orphaned tool result is an invalid sequence for every provider.
- while (tail_start < len(history) and isinstance(history[tail_start], dict)
- and history[tail_start].get("role") == "tool"):
- tail_start += 1
- return history[:head_end] + history[tail_start:]
- def hygiene_no_commit_reason(agent) -> str:
- """Name WHY a hygiene compression left the session id unchanged with no in-place commit.
- The terminal ``else`` used to blame "no session_db on the hygiene agent" for every route into it,
- but that is one of several causes (#71097): an attempt that ABORTED before any commit boundary
- (lock skip, transient cooldown, summary timeout, codex thread interrupted) leaves
- ``_last_compression_attempt_in_place`` at ``None``; a DB-less agent is only the case when
- ``_session_db`` really is missing. Read the per-attempt signals the compressor sets, in that order."""
- if not bool(getattr(agent, "_last_compression_attempt_recorded", False)):
- return "compression did not run"
- lock_skip = getattr(agent, "_compression_skipped_due_to_lock", None)
- if lock_skip is True or isinstance(lock_skip, str):
- return "attempt skipped: compression lease held by another process"
- blocked = getattr(agent, "_compression_blocked_transient", None)
- if blocked:
- return f"attempt blocked: {blocked}"
- if getattr(agent, "_last_compression_attempt_in_place", None) is None:
- detail = "summary timed out" if getattr(agent, "_last_compression_timed_out", False) else "aborted before commit"
- warning = getattr(agent, "_last_compression_summary_warning", None)
- return f"attempt {detail}" + (f": {warning}" if warning else "")
- if getattr(agent, "_session_db", None) is None:
- return "no session_db on the hygiene agent"
- return "in-place commit did not complete"
- from gateway.run_turn_prepare import GatewayTurnPrepareMixin
- from gateway.run_background import GatewayBackgroundMixin
- from gateway.run_turn_hygiene import GatewayTurnHygieneMixin
- from gateway.run_turn_persistence import GatewayTurnPersistenceMixin
- class GatewayTurnMixin(GatewayBackgroundMixin, GatewayTurnPrepareMixin, GatewayTurnHygieneMixin, GatewayTurnPersistenceMixin):
- """Agent-turn execution for GatewayRunner (see module docstring)."""
- def _event_thread_metadata(self, event, source):
- """Thread metadata for a send that replies to ``event`` on ``source``."""
- return self._thread_metadata_for_source(source, self._reply_anchor_for_event(event))
- @staticmethod
- def _pop_post_delivery_callback(adapter, key, generation):
- """Pop the adapter's deferred post-delivery callback for ``key`` (legacy dict fallback)."""
- if getattr(type(adapter), "pop_post_delivery_callback", None) is not None:
- return adapter.pop_post_delivery_callback(key, generation=generation)
- if adapter and hasattr(adapter, "_post_delivery_callbacks"):
- return adapter._post_delivery_callbacks.pop(key, None)
- return None
- async def _hmwa_stop_typing_for_turn(self, event, source):
- """Stop the typing indicator (never raises), with the delivery path's thread/workspace routing
- metadata (Slack AI status is thread-scoped), on this turn's own message, not the chat's latest."""
- from gateway.platforms.helpers import typing_metadata_for_turn
- with suppress(Exception):
- _typing_adapter = self._delivery_adapter_for(source)
- _kind = type(_typing_adapter)
- if _typing_adapter and callable(getattr(_kind, "_stop_typing_with_metadata", None)):
- await _typing_adapter._stop_typing_with_metadata(source.chat_id, typing_metadata_for_turn(_typing_adapter,
- source.chat_id, self._reply_anchor_for_event(event), self._event_thread_metadata(event, source)))
- elif _typing_adapter and callable(getattr(_kind, "stop_typing", None)):
- await _typing_adapter.stop_typing(source.chat_id)
- async def _handle_message_with_agent(self, event, source, _quick_key: str, run_generation: int):
- """Inner handler that runs under the _running_agents sentinel guard."""
- _msg_start_time = time.time()
- _platform_name = source.platform.value if hasattr(source.platform, "value") else str(source.platform)
- logger.info(
- "inbound message: platform=%s user=%s chat=%s msg=%r reply_to_id=%s reply_to_text=%r",
- _platform_name, source.user_name or source.user_id or "unknown",
- source.chat_id or "unknown", (event.text or "")[:80].replace("\n", " "),
- getattr(event, "reply_to_message_id", None),
- (getattr(event, "reply_to_text", None) or "")[:80].replace("\n", " "),
- )
- resolved = await self._hmwa_resolve_session(event, source)
- if resolved is None:
- return
- source, session_entry, session_key = resolved
- prepared, _session_env_tokens = await self._hmwa_prepare_turn(
- event, source, session_entry, session_key, _quick_key, run_generation,
- )
- if not isinstance(prepared, self._PreparedTurn):
- return prepared
- history, message_text = prepared.history, prepared.message_text
- try:
- hook_ctx = {
- "platform": source.platform.value if source.platform else "",
- "user_id": source.user_id,
- "chat_id": source.chat_id or "",
- "thread_id": str(source.thread_id) if getattr(source, "thread_id", None) else "",
- "chat_type": getattr(source, "chat_type", "") or "",
- "session_id": session_entry.session_id,
- "message": message_text[:500],
- }
- await self.hooks.emit("agent:start", hook_ctx)
- # Capture the launch session id so post-run compression publication is identity-guarded
- # (a /new may move session_entry.session_id while the old run is still unwinding).
- from gateway.run_heartbeat_acceptance import heartbeat_owner_is_current
- if not heartbeat_owner_is_current(self, event, session_key):
- return
- _run_start_session_id = session_entry.session_id
- _turn_started_monotonic = time.monotonic()
- # Admission/typing is not execution. All routing, authorization and
- # turn preparation gates have passed when the agent runner is entered.
- event._heartbeat_execution_started = True
- _turn_channel_prompt, _turn_source = self._pinned_channel_inputs( # internal/synthetic reuse pins
- session_key, event.channel_prompt, source,
- internal=event.internal or getattr(event, "inherit_channel_pin", False))
- if not event.internal:
- # Persist the coherent context+channel pair before execution: a crash during the
- # human turn may be followed by an internal startup-resume on the next process.
- await self._persist_prompt_pins(session_key, _run_start_session_id)
- agent_result = await self._run_agent(
- message=message_text, context_prompt=prepared.context_prompt, history=history, source=_turn_source,
- session_id=_run_start_session_id, session_key=session_key,
- run_generation=run_generation, event_message_id=self._reply_anchor_for_event(event),
- inbound_message_id=str(event.message_id) if event.message_id else None,
- channel_prompt=_turn_channel_prompt, moa_config=getattr(event, "_moa_config", None),
- title_user_message=prepared.title_user_message,
- persist_user_message=prepared.persist_user_message,
- persist_user_timestamp=prepared.persist_user_timestamp,
- persist_user_display_kind=prepared.persist_user_display_kind,
- reply_expected=event.reply_expected,
- persist_user_display_metadata={
- "gateway_input_owner": prepared.persistence_owner, **persisted_user_metadata_for_event(event),
- **reply_expected_metadata(event.reply_expected), **diagnostic_metadata(event)},
- message_type=event.message_type,
- scheduled_heartbeat=bool(getattr(event, "_heartbeat_session_id", None)),
- )
- _turn_seconds = time.monotonic() - _turn_started_monotonic
- # A queued (/queue) chain answered the LAST message of the chain, so the outer final
- # send (bracketed by the adapter against this event) must be ledgered under that
- # message's id or it collides with an earlier turn's row carrying the same text. Reply
- # routing is untouched: the anchor still comes from this event.
- if isinstance(agent_result, dict):
- _terminal_inbound = agent_result.get("queued_terminal_inbound_id")
- if _terminal_inbound:
- event.ledger_message_id = str(_terminal_inbound)
- if "queued_terminal_notification_category" in agent_result:
- event.metadata["notification_category"] = agent_result["queued_terminal_notification_category"]
- if isinstance(agent_result.get("_notification_reply_muted"), bool):
- event._notification_reply_muted = agent_result["_notification_reply_muted"]
- await self._hmwa_stop_typing_for_turn(event, source)
- if not self._is_session_run_current(_quick_key, run_generation):
- self._hmwa_discard_stale_result(source, _quick_key, run_generation)
- return None
- response, _intentional_silence, agent_messages = await self._hmwa_shape_agent_response(
- agent_result, source, history, session_entry, session_key,
- _quick_key, run_generation, _run_start_session_id, _platform_name, _msg_start_time,
- persist_user_display_kind=prepared.persist_user_display_kind,
- reply_expected=event.reply_expected,
- )
- response = self._usage_window_chat_wording(agent_result, self._hmwa_prepend_reasoning(agent_result, response, source, _intentional_silence))
- _footer_line = self._hmwa_runtime_footer_line(agent_result, source, _turn_seconds)
- # Streaming already delivered the body: the footer goes out as a trailing send instead.
- if _footer_line and response and not agent_result.get("already_sent") and not _intentional_silence:
- response = f"{response}\n\n{_footer_line}"
- await self._hmwa_post_turn_hooks(hook_ctx, agent_result, response)
- agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure = (
- self._hmwa_classify_turn_failure(agent_result, history, session_entry)
- )
- if agent_failed_early and not is_context_overflow_failure and automatic_notices_allowed(source):
- response = self._hmwa_add_failed_turn_notice(response, self._hmwa_failed_turn_notice(agent_result))
- response, session_entry = await self._hmwa_compression_exhaustion_reset(
- agent_result, response, session_entry, session_key, source,
- )
- await self._hmwa_persist_turn_transcript(
- event=event, source=source, session_entry=session_entry, session_key=session_key,
- agent_result=agent_result, agent_messages=agent_messages, prepared=prepared,
- response=response, agent_failed_early=agent_failed_early,
- hidden_reasoning_incomplete=hidden_reasoning_incomplete,
- is_context_overflow_failure=is_context_overflow_failure,
- )
- await self._record_room_silence_decision(
- session_key, agent_result, _intentional_silence,
- display_kind=prepared.persist_user_display_kind, reply_expected=event.reply_expected)
- return await self._deliver_turn_response_then_limit_offer(
- event, source, session_entry, session_key, run_generation,
- agent_result, agent_messages, response, _footer_line, _intentional_silence,
- )
- except Exception as e:
- return await self._hmwa_agent_error_reply(e, event, source, session_entry, session_key, prepared)
- finally:
- # Restore session context variables to their pre-handler state
- self._clear_session_env(_session_env_tokens)
- def _profile_scope_for_source(self, source: SessionSource):
- """``_profile_runtime_scope`` for ``source``'s profile when a secret scope is required.
- Under multiplexing config/skills/memory resolve to the source profile's home AND credentials
- come from its secret scope (never process-global ``os.environ``). A standalone gateway
- (``multiplex_profiles`` off) still binds once a hosted room has flipped the process-wide
- credential guard — see ``_standalone_launch_scope``."""
- from gateway.run import _profile_runtime_scope
- from gateway.session_policy import local_desktop_turn
- home = self._profile_scope_key_for_source(source)
- return local_desktop_turn(self, source, _profile_runtime_scope(home) if home is not None
- else self._standalone_launch_scope())
- def _profile_scope_key_for_source(self, source: SessionSource) -> Optional[Path]:
- """Profile home ``_profile_scope_for_source`` binds for ``source``, or ``None`` when it falls
- back to the standalone launch scope. The single owner of that branch condition: callers that
- group work per scope (heartbeat restore) key on this so they cannot drift from the scope
- actually entered."""
- if getattr(getattr(self, "config", None), "multiplex_profiles", False):
- return self._resolve_profile_home_for_source(source)
- return None
- def _async_profile_scope_for_source(self, source: SessionSource):
- """``async with`` twin of :meth:`_profile_scope_for_source` (secret hydration off-loop).
- Slash dispatch runs under the RECEIVING bot's scope (auth needs its ``.env``), which is not
- the routed runtime when a bot serves another profile's chat; every handler reading
- home-relative state (pending writes, memory store, config) binds the runtime here (#119915)."""
- from gateway.run import _async_profile_runtime_scope
- home = self._profile_scope_key_for_source(source)
- if home is not None:
- return _async_profile_runtime_scope(home)
- from tui_gateway.launch_profile_policy import async_launch_profile_scope_if_multiplexed
- return async_launch_profile_scope_if_multiplexed()
- @staticmethod
- def _standalone_launch_scope():
- """Scope for a standalone gateway's own (launch-profile) work: a no-op until the process hosts
- another profile home, then the launch profile's OWN runtime scope.
- A native hosted room running a second profile calls
- ``tui_gateway.launch_profile_policy.activate_multi_profile_hosting`` inside the gateway process,
- so ``get_secret`` fails closed for every unscoped read afterwards — including the standalone
- gateway's ordinary turns, which never bound a scope because ``multiplex_profiles`` is off
- (#112878). The launch profile is a profile too: bind its ``.env`` over the env frozen at
- activation (a key injected by systemd / ``op run`` has no file to rebuild it from), never a
- secondary's scope and never live ``os.environ``."""
- from tui_gateway.launch_profile_policy import launch_profile_scope_if_multiplexed
- return launch_profile_scope_if_multiplexed()
- def _media_delivery_scope_for_source(self, source: SessionSource):
- """Home + terminal-policy scope for validating a turn's MEDIA / local-file paths on the
- adapter's delivery side, which runs after the routed turn scope was reset.
- Docker path translation (``platforms/base.py::_translate_docker_container_media_path``)
- infers the producing container from the ACTIVE profile (``get_active_profile_name``) and the
- scope-aware ``TERMINAL_DOCKER_VOLUMES``; without this a secondary's ``MEDIA:/output/x.png``
- resolves against the default profile's sandbox and mounts (#109024). No secret hydration:
- path validation reads no credentials and this runs on the event loop."""
- if not getattr(getattr(self, "config", None), "multiplex_profiles", False):
- return nullcontext()
- from gateway.run import _profile_runtime_scope
- return _profile_runtime_scope(self._resolve_profile_home_for_source(source), {})
- def _reset_notice_session_info(self, source: SessionSource) -> str:
- """Session-info block for the auto-reset notice, resolved inside the profile serving ``source``.
- Call via ``asyncio.to_thread``: resolution can block (credential refresh, context-length
- probes), and the scope is entered here so contextvars behave in the worker thread."""
- with self._profile_scope_for_source(source):
- return self._format_session_info()
- def _format_session_info(self) -> str:
- """Model / provider / context-length / endpoint block so users can spot bad context detection."""
- from gateway.run import _resolve_gateway_model_context
- resolved = _resolve_gateway_model_context()
- context_length = resolved.context_length
- ctx_source = {
- "config": "config",
- "default": "fallback, not a detected limit — set model.context_length in config to override",
- }.get(resolved.context_source, "detected")
- ctx_display = (
- f"{context_length / 1_000_000:.1f}M" if context_length >= 1_000_000
- else f"{context_length // 1_000}K" if context_length >= 1_000 else str(context_length)
- )
- lines = [
- t("gateway.session.info_model", model=resolved.model),
- t("gateway.session.info_provider", provider=resolved.provider or "openrouter"),
- t("gateway.session.info_context", tokens=ctx_display, source=ctx_source),
- ]
- if (resolved.provider or "") == "moa":
- # The preset name hides who pays: the aggregator runs every tool-loop step (#112359).
- from hermes_cli.config import load_config
- from hermes_cli.moa_config import normalize_moa_config
- agg = normalize_moa_config(load_config().get("moa"))["presets"].get(resolved.model, {}).get("aggregator") or {}
- if agg:
- lines.append(t("gateway.session.info_acting_model", provider=agg.get("provider"), model=agg.get("model")))
- base_url = resolved.base_url
- if base_url and base_url_hostname(base_url) in ("localhost", "127.0.0.1", "0.0.0.0"):
- lines.append(t("gateway.session.info_endpoint", url=base_url))
- return "\n".join(lines)
- def _resolve_enabled_toolsets_for_source(
- self, user_config: dict, source: SessionSource, platform_key: str,
- ) -> list:
- """Enabled toolsets for an agent run, honoring an adapter ``toolsets_for_source()`` override
- validated through the SAME ``_get_platform_tools`` path (unknown / platform-restricted
- toolsets dropped, not trusted)."""
- from hermes_cli.tools_config import _get_platform_tools
- try:
- adapter = self._delivery_adapter_for(source)
- override = adapter.toolsets_for_source(source) if adapter is not None else None
- except Exception:
- override = None
- if override and isinstance(override, list):
- pts = dict(user_config.get("platform_toolsets") or {})
- pts[platform_key] = [str(x) for x in override]
- user_config = {**user_config, "platform_toolsets": pts}
- return sorted(_get_platform_tools(user_config, platform_key))
- def _resolve_turn_toolsets(self, user_config: dict, source: SessionSource, platform_key: str):
- """``(enabled_toolsets, disabled_toolsets)`` for an agent run on ``source``."""
- from agent.skill_utils import parse_config_string_list
- enabled = self._resolve_enabled_toolsets_for_source(user_config, source, platform_key)
- disabled = parse_config_string_list((user_config.get("agent") or {}).get("disabled_toolsets")) or None
- return enabled, disabled
- def _mcp_reload_refresh_cached_agents(self, multiplex: bool, profile) -> None:
- """Refresh cached agents so existing sessions see new MCP tools on their next turn without
- a history-destroying ``/new``. Each agent keeps its build-time toolset selection EXACTLY: a
- session built with restricted enabled_toolsets (e.g. ["safe"]) must NOT silently gain tools."""
- try:
- from tools.mcp_tool_agent import refresh_agent_mcp_tools
- _cache = getattr(self, "_agent_cache", None)
- _cache_lock = getattr(self, "_agent_cache_lock", None)
- if _cache_lock is None or not _cache:
- return
- # Multiplex: only this profile's sessions (another profile's agent would get this registry).
- _ns_prefix = _session_key_namespace(profile) + ":" if multiplex else None
- with _cache_lock:
- for _sess_key, _entry in list(_cache.items()):
- if _ns_prefix and not str(_sess_key).startswith(_ns_prefix):
- continue
- _agent = _entry[0] if isinstance(_entry, tuple) else _entry
- if _agent is not None:
- refresh_agent_mcp_tools(_agent, quiet_mode=True)
- except Exception as _exc:
- logger.debug("Failed to update cached agent tools after MCP reload: %s", _exc)
- async def _execute_mcp_reload(self, event: MessageEvent) -> str:
- """Disconnect, reconnect, and notify MCP tool changes (shared by button / text / no-confirm paths).
- Under multiplex the reload runs inside the requesting profile's runtime scope (entered here
- when the caller did not) and only that profile's servers are torn down and rediscovered.
- See #95518.
- """
- from gateway.run import _profile_runtime_scope
- multiplex = bool(getattr(self.config, "multiplex_profiles", False))
- if multiplex and not get_hermes_home_override():
- profile_home = self._resolve_profile_home_for_source(event.source)
- with _profile_runtime_scope(Path(profile_home)):
- return await self._execute_mcp_reload(event)
- try:
- from tools.mcp_tool_lifecycle import shutdown_mcp_servers
- from tools.mcp_tool_discovery import discover_mcp_tools
- from tools.mcp_tool import _servers, _lock, _server_visible_in_scope
- from tools.mcp_tool_agent import reprobe_tool_availability
- from tools.mcp_tool_scope import _key_name
- from tools.registry import registry
- reload_scope = registry.current_scope_key() if multiplex else None
- def _scoped_server_names() -> set:
- with _lock:
- return {_key_name(key) for key in _servers if _server_visible_in_scope(key, reload_scope)}
- old_servers = _scoped_server_names()
- await self._run_in_executor_with_context(lambda: shutdown_mcp_servers(scope=reload_scope))
- # Explicit reload also re-probes tool availability (check_fn).
- reprobe_tool_availability()
- # Reconnect by discovering tools (reads config.yaml fresh). A chat command cannot finish
- # a browser OAuth flow either: an expired token parks with a `hermes mcp login` hint.
- from tools.mcp_oauth import suppress_interactive_oauth
- with suppress_interactive_oauth():
- new_tools = await self._run_in_executor_with_context(discover_mcp_tools)
- connected_servers = _scoped_server_names()
- if reload_scope is not None:
- from tools.mcp_tool import _mcp_tool_server_names
- with _lock:
- new_tools = [n for n in new_tools if _mcp_tool_server_names.get(n) in connected_servers]
- # (label, i18n key, names); i18n lines list reconnected first, the injected note added first.
- changes = (
- ("Reconnected", "gateway.reload_mcp.reconnected", connected_servers & old_servers),
- ("Added", "gateway.reload_mcp.added", connected_servers - old_servers),
- ("Removed", "gateway.reload_mcp.removed", old_servers - connected_servers),
- )
- lines = [t("gateway.reload_mcp.header")] + [
- t(key, names=", ".join(sorted(names))) for _label, key, names in changes if names
- ]
- from tools.mcp_tool_discovery import mcp_servers_not_connected
- failed = await self._run_in_executor_with_context(mcp_servers_not_connected)
- lines.append(t("gateway.reload_mcp.none_connected") if not connected_servers else t(
- "gateway.reload_mcp.tools_available", tools=len(new_tools), servers=len(connected_servers)))
- if failed: # a configured server that did not connect is named, never passed over as success
- lines.append(t("gateway.reload_mcp.failed", error=f"did not connect: {', '.join(failed)}"))
- self._mcp_reload_refresh_cached_agents(multiplex, event.source.profile)
- # Append a note at the END of the history (preserves the prompt-cache prefix).
- change_parts = [
- f"{label} servers: {', '.join(sorted(names))}"
- for label, _key, names in (changes[1], changes[2], changes[0]) if names
- ]
- tool_summary = f"{len(new_tools)} MCP tool(s) now available" if new_tools else "No MCP tools available"
- change_detail = ". ".join(change_parts) + ". " if change_parts else ""
- reload_msg = {
- "role": "user",
- "content": f"[IMPORTANT: MCP servers have been reloaded. {change_detail}{tool_summary}. The tool list for this conversation has been updated accordingly.]",
- }
- with suppress(Exception): # Best-effort; don't fail the reload over a transcript write
- session_entry = await self.async_session_store.get_or_create_session(event.source)
- await self.async_session_store.append_to_transcript(session_entry.session_id, reload_msg)
- return "\n".join(lines)
- except Exception as e:
- logger.warning("MCP reload failed: %s", e)
- return t("gateway.reload_mcp.failed", error=e)
- def _get_proxy_url(self) -> Optional[str]:
- """Proxy URL if proxy mode is configured (GATEWAY_PROXY_URL env wins over ``gateway.proxy_url``).
- Per-profile like GATEWAY_PROXY_KEY: under multiplex a raw environ read would ship a secondary's
- turns (authenticated with ITS scoped key) to the default profile's proxy. Same fallback shape as
- the key — only ``UnscopedSecretError`` (the unscoped default-profile path) reads the env."""
- from gateway.run import _load_gateway_config
- from agent.secret_scope import UnscopedSecretError, get_secret
- try:
- url = (get_secret("GATEWAY_PROXY_URL") or "").strip()
- except UnscopedSecretError:
- url = os.getenv("GATEWAY_PROXY_URL", "").strip()
- if not url:
- url = ((_load_gateway_config().get("gateway") or {}).get("proxy_url") or "").strip()
- return url.rstrip("/") if url else None
- def _build_stream_consumer_config(
- self, source: SessionSource, scfg: Any, adapter: Any, *, on_missing_cursor: str,
- ) -> tuple[Any, Optional[Callable[[], None]]]:
- """Build the shared ``StreamConsumerConfig`` and optional Telegram pause-typing closure.
- For non-editing adapters ``on_missing_cursor="fallback"`` streams with an empty cursor;
- ``"raise"`` raises ``RuntimeError`` so the caller skips streaming entirely."""
- from gateway.stream_consumer import StreamConsumerConfig
- _pause_typing_before_finalize = None
- if source.platform == Platform.TELEGRAM and hasattr(adapter, "pause_typing_for_chat"):
- def _pause_typing_before_finalize(_adapter=adapter, _chat_id=source.chat_id) -> None:
- _adapter.pause_typing_for_chat(_chat_id)
- # Non-editing platforms (QQ, WeChat) skip streaming — the partial first message could never
- # be updated — unless they have a native-streaming transport (WeCom msgtype "stream").
- _adapter_supports_edit = getattr(adapter, "SUPPORTS_MESSAGE_EDITING", True)
- _adapter_supports_native_stream = bool(getattr(adapter, "SUPPORTS_NATIVE_STREAMING", False))
- if not _adapter_supports_edit and not _adapter_supports_native_stream and on_missing_cursor == "raise":
- raise RuntimeError("skip streaming for non-editable platform")
- _effective_cursor = scfg.cursor if _adapter_supports_edit else ""
- # Some Matrix clients render the cursor as tofu: stream text, no cursor.
- if source.platform == Platform.MATRIX:
- _effective_cursor = ""
- # Fresh-final applies to Telegram only (others edit in place cheaply).
- # Fresh-final applies to Telegram only — other platforms either edit in place cheaply (Discord,
- # Slack) or don't have the timestamp-on-edit / edit-timestamp-stays-stale problem. (Ported from
- # openclaw/openclaw#72038.)
- _fresh_final_secs = (
- float(getattr(scfg, "fresh_final_after_seconds", 0.0) or 0.0)
- if source.platform == Platform.TELEGRAM else 0.0
- )
- _consumer_cfg = StreamConsumerConfig(
- edit_interval=scfg.edit_interval, buffer_threshold=scfg.buffer_threshold,
- cursor=_effective_cursor,
- fresh_final_after_seconds=_fresh_final_secs, transport=scfg.transport or "edit",
- chat_type=getattr(source, "chat_type", "") or "",
- )
- return _consumer_cfg, _pause_typing_before_finalize
- def _run_still_current_fn(self, session_key: Optional[str], run_generation: Optional[int]) -> Callable[[], bool]:
- """Predicate: does this run's generation still own ``session_key``? (always True when untracked)."""
- def _run_still_current() -> bool:
- if run_generation is None or not session_key:
- return True
- return self._is_session_run_current(session_key, run_generation)
- return _run_still_current
- @staticmethod
- def _proxy_error_result(text: str) -> dict[str, Any]:
- return {"final_response": text, "messages": [], "api_calls": 0, "tools": []}
- def _proxy_stream_consumer(self, source: SessionSource, event_message_id, _thread_metadata, _run_still_current):
- """Platform stream consumer for the proxy path when streaming is enabled, else ``None``."""
- from gateway.run import _load_gateway_config, _platform_config_key
- _scfg = getattr(getattr(self, "config", None), "streaming", None)
- # #60671 — streaming TTS consumer is created on the outer event-loop thread before run_sync
- # launches. run_sync only reads it via ``streaming_tts_consumer_holder[0]`` for delta callback
- # wiring.
- if _scfg is None:
- from gateway.config import StreamingConfig
- _scfg = StreamingConfig()
- # Global master switch first: skips the config.yaml re-read on the default (off) path.
- if not _scfg.globally_enabled:
- return None
- from gateway.display_config import resolve_display_setting
- _plat_streaming = resolve_display_setting(_load_gateway_config(), _platform_config_key(source.platform), "streaming")
- if not _scfg.enabled_for(_plat_streaming):
- return None
- try:
- from gateway.stream_consumer import GatewayStreamConsumer
- _adapter = self._delivery_adapter_for(source)
- if not _adapter:
- return None
- _consumer_cfg, _pause_typing_before_finalize = self._build_stream_consumer_config(
- source, _scfg, _adapter, on_missing_cursor="fallback",
- )
- return GatewayStreamConsumer(
- adapter=_adapter, chat_id=source.chat_id, config=_consumer_cfg,
- metadata=_thread_metadata, on_before_finalize=_pause_typing_before_finalize,
- initial_reply_to_id=event_message_id, run_still_current=_run_still_current,
- )
- except Exception as _sc_err:
- logger.debug("Proxy: could not set up stream consumer: %s", _sc_err)
- return None
- async def _run_agent_via_proxy(
- self, message: str, context_prompt: str, history: list[dict[str, Any]],
- source: SessionSource, session_id: str, session_key: str | None = None,
- run_generation: Optional[int] = None, event_message_id: Optional[str] = None,
- scheduled_heartbeat: bool = False,
- ) -> dict[str, Any]:
- """Forward the message to a remote Hermes API server instead of running a local AIAgent.
- Lets a Docker container handle Matrix E2EE while the agent runs on the host with full
- access to local files, memory, skills, and a unified session store."""
- from gateway.run import _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS
- try:
- from aiohttp import ClientSession as _AioClientSession, ClientTimeout
- except ImportError:
- return self._proxy_error_result(t("gateway.proxy.requires_aiohttp", hint=install_hint("messaging")))
- proxy_url = self._get_proxy_url()
- if not proxy_url:
- return self._proxy_error_result(t("gateway.proxy.url_missing"))
- # The proxy key is a per-profile credential read through the installed secret scope:
- # ``_run_agent`` binds the source profile's scope (or the launch profile's once the process
- # multiplexes), and an unscoped single-profile process reads os.environ inside get_secret.
- # Any get_secret() error propagates rather than degrading to the ambient key, which may hold
- # another profile's credential.
- from agent.secret_scope import get_secret_str
- proxy_key = get_secret_str("GATEWAY_PROXY_KEY").strip()
- _run_still_current = self._run_still_current_fn(session_key, run_generation)
- def _stale_result(what: str) -> dict[str, Any]:
- logger.info(
- "Discarding stale proxy %s for %s — generation %d is no longer current",
- what, session_key or "?", run_generation or 0,
- )
- return {
- "final_response": "", "messages": [], "api_calls": 0, "tools": [],
- "history_offset": len(history), "session_id": session_id, "response_previewed": False,
- }
- # OpenAI chat format. The remote keeps continuity via X-Hermes-Session-Id; send the current
- # message plus a compact text-only history for a remote that has none yet.
- api_messages: list[dict[str, str]] = [{"role": "system", "content": context_prompt}] if context_prompt else []
- api_messages += [
- {"role": msg.get("role"), "content": msg.get("content")}
- for msg in history if msg.get("role") in {"user", "assistant"} and msg.get("content")
- ]
- api_messages.append({"role": "user", "content": message})
- headers: dict[str, str] = {"Content-Type": "application/json"}
- if proxy_key:
- headers["Authorization"] = f"Bearer {proxy_key}"
- if session_id:
- headers["X-Hermes-Session-Id"] = session_id
- body = {"model": "hermes-agent", "messages": api_messages, "stream": True}
- _thread_metadata: Optional[dict[str, Any]] = self._thread_metadata_for_source(source, event_message_id)
- _stream_consumer = (
- None if scheduled_heartbeat
- else self._proxy_stream_consumer(source, event_message_id, _thread_metadata, _run_still_current)
- )
- stream_task = asyncio.create_task(_stream_consumer.run()) if _stream_consumer else None
- _adapter = self._delivery_adapter_for(source)
- if _adapter and not scheduled_heartbeat:
- with suppress(Exception):
- await _adapter.send_typing(source.chat_id, metadata=typing_metadata_for_turn(
- _adapter, source.chat_id, event_message_id, _thread_metadata))
- full_response = ""
- _start = time.time()
- saw_done = False
- def _consume_sse_line(line: str) -> bool:
- """Parse one SSE line into full_response; True when the terminal ``[DONE]`` was seen.
- Malformed frames (bad JSON, ``choices: [null]``, non-dict deltas) are skipped —
- one bad chunk must not abort the whole stream."""
- nonlocal full_response
- line = line.strip()
- if not line.startswith("data: "):
- return False
- data = line[6:]
- if data.strip() == "[DONE]":
- return True
- try:
- choices = json.loads(data).get("choices") or []
- content = choices[0].get("delta", {}).get("content", "") if choices else ""
- except (json.JSONDecodeError, TypeError, AttributeError, IndexError):
- return False
- if content:
- full_response += content
- if _stream_consumer:
- _stream_consumer.on_delta(content)
- return False
- try:
- # sock_connect bounds the TCP connect phase so an unreachable proxy host
- # (DNS fail, firewall, remote down) fails fast instead of hanging on the OS default.
- _timeout = ClientTimeout(total=0, sock_read=1800, sock_connect=30)
- async with _AioClientSession(timeout=_timeout) as session:
- async with session.post(f"{proxy_url}/v1/chat/completions", json=body, headers=headers) as resp:
- if resp.status != 200:
- error_text = await resp.text()
- logger.warning("Proxy error (%d) from %s: %s", resp.status, proxy_url, error_text[:500])
- return self._proxy_error_result(t("gateway.proxy.http_error", status=resp.status, error=error_text[:300]))
- buffer = ""
- async for chunk in resp.content.iter_any():
- if not _run_still_current():
- return _stale_result("stream")
- buffer += chunk.decode("utf-8", errors="replace")
- while "\n" in buffer:
- line, buffer = buffer.split("\n", 1)
- if _consume_sse_line(line):
- saw_done = True
- break
- # Exit before the iterator requests another network chunk.
- if saw_done:
- break
- if len(buffer) > _GATEWAY_PROXY_SSE_BUFFER_MAX_CHARS:
- raise ValueError("Proxy SSE stream exceeded max buffer size without a line boundary")
- # The final SSE frame may not be newline-terminated: flush the residual
- # buffer after EOF instead of silently dropping its content.
- if not saw_done and buffer:
- saw_done = _consume_sse_line(buffer)
- if not saw_done:
- # Clean EOF without [DONE] — the upstream dropped the response
- # mid-stream. Keep any partial text but say so instead of
- # presenting the truncation as a complete answer.
- logger.warning(
- "Proxy SSE stream from %s ended without [DONE] — response may be truncated "
- "(%d chars received)", proxy_url, len(full_response),
- )
- if not full_response:
- return self._proxy_error_result(t("gateway.proxy.closed_early"))
- except asyncio.CancelledError:
- raise
- except Exception as e:
- logger.error("Proxy connection error to %s: %s", proxy_url, e)
- if not full_response:
- return self._proxy_error_result(t("gateway.proxy.connection_error", error=e))
- # Partial response — return what we got
- finally:
- if _stream_consumer:
- _stream_consumer.finish()
- if stream_task:
- try:
- await asyncio.wait_for(stream_task, timeout=5.0)
- except (TimeoutError, asyncio.CancelledError):
- stream_task.cancel()
- _elapsed = time.time() - _start
- if not _run_still_current():
- return _stale_result("result")
- logger.info(
- "proxy response: url=%s session=%s time=%.1fs response=%d chars",
- proxy_url, (session_id or "")[:20], _elapsed, len(full_response),
- )
- return {
- "final_response": full_response or t("gateway.proxy.no_response"),
- "messages": [
- {"role": "user", "content": message},
- {"role": "assistant", "content": full_response},
- ],
- "api_calls": 1,
- "tools": [],
- "history_offset": len(history),
- "session_id": session_id,
- "response_previewed": _stream_consumer is not None and bool(full_response),
- }
- async def _run_agent(
- self, message: str, context_prompt: str, history: list[dict[str, Any]],
- source: SessionSource, session_id: str, **turn_kwargs,
- ) -> dict[str, Any]:
- """Profile-scoping wrapper around ``_run_agent_inner`` (same keyword parameters; pass-through
- when multiplexing is off)."""
- with self._profile_scope_for_source(source):
- return await self._run_agent_inner(message, context_prompt, history, source, session_id, **turn_kwargs)
- def _run_agent_display_settings(self, source: SessionSource) -> GatewayRunner._RunAgentDisplay:
- """Resolve per-platform display, progress, status and streaming-surface settings for a turn."""
- from gateway.run import (
- _gateway_platform_value, _has_platform_display_override, _load_gateway_config,
- _platform_config_key,
- )
- from agent.secret_scope import get_secret
- from gateway.display_config import resolve_display_setting, resolve_tool_progress
- from gateway.status_phrases import choose_status_phrase, resolve_status_phrase_catalog
- from gateway.session_policy import policy_for_source, policy_turn_toolsets
- policy = policy_for_source(self, source)
- from gateway.session_authorities import active_authority
- user_config = policy.config(active_authority(self), secrets=False) if policy else _load_gateway_config()
- platform_key = policy.platform if policy else _platform_config_key(source.platform)
- enabled_toolsets, disabled_toolsets = self._resolve_turn_toolsets(user_config, source, platform_key)
- if policy:
- enabled_toolsets = policy_turn_toolsets(active_authority(self), policy, user_config, source.chat_id)
- adapter = self._delivery_adapter_for(source)
- # Tool preview length (0 = no limit) and friendly tool labels (default on), per-platform.
- for _setter, _setting, _default, _cast in (
- ("set_tool_preview_max_len", "tool_preview_length", 0, lambda v: int(v) if v else 0),
- ("set_friendly_tool_labels", "friendly_tool_labels", True, bool),
- ):
- with suppress(Exception):
- from agent import display as _agent_display
- _val = resolve_display_setting(user_config, platform_key, _setting, _default)
- getattr(_agent_display, _setter)(_cast(_val))
- # Resolve the mode and its provenance together: null inherits, tier off is not intent.
- # A raw os.getenv here reads whichever profile's env loaded last under multiplexing
- # (#116898); get_secret resolves through the active profile's scope instead.
- progress_mode, _tool_progress_explicit = resolve_tool_progress(
- user_config, platform_key, get_secret("HERMES_TOOL_PROGRESS_MODE"),
- )
- # "accumulate" (edit one bubble) or "separate" (one msg per tool)
- progress_grouping = resolve_display_setting(user_config, platform_key, "tool_progress_grouping") or "accumulate"
- _generic_status_recent: list[str] = []
- _generic_status_catalog = resolve_status_phrase_catalog(user_config, platform_key)
- def _display_surface_mode(
- setting: str, *, default: bool = False,
- require_platform_override_for: set[Any] | None = None, allow_generic: bool = False,
- ) -> str:
- """Return off|raw|generic for a gateway visibility surface."""
- if require_platform_override_for:
- current_platform = _gateway_platform_value(source.platform)
- platform_only = {_gateway_platform_value(item) for item in require_platform_override_for}
- if (
- current_platform in platform_only
- and not _has_platform_display_override(user_config, platform_key, setting)
- ):
- return "off"
- value = resolve_display_setting(user_config, platform_key, setting, default)
- if isinstance(value, str) and value.strip().lower() == "generic":
- return "generic" if allow_generic else "off"
- return "raw" if bool(value) else "off"
- def _generic_status_phrase(kind: str, *, tool_name: str | None = None, preview: str | None = None, args: Any = None) -> str:
- try:
- return choose_status_phrase(
- kind, tool_name=tool_name, preview=preview, args=args,
- recent=_generic_status_recent, catalog=_generic_status_catalog,
- )
- except Exception as _phrase_err:
- logger.debug("generic status phrase selection failed: %s", _phrase_err)
- return (t("gateway.progress.status_fallback_long")
- if kind in {"heartbeat", "waiting", "long_running", "status"}
- else t("gateway.progress.status_fallback_short"))
- # Webhooks can't edit messages; a bot peer gets the final reply only (#137027): no progress lanes.
- is_webhook, bot_peer = source.platform == Platform.WEBHOOK, not automatic_notices_allowed(source)
- tool_progress_enabled = progress_mode not in {"off", "log"} and not is_webhook and not bot_peer
- # Live status for text-rendering typing indicators (Slack); independent of tool_progress.
- _live_status_mode = resolve_display_setting(user_config, platform_key, "live_status", "full")
- _live_status_adapter = (
- adapter if getattr(adapter, "supports_status_text", False) and _live_status_mode != "off" else None
- )
- # "log" mode: tool calls go to ~/.hermes/logs/tool_calls.log instead of the chat. Gateway-only.
- log_mode_enabled = progress_mode == "log" and not is_webhook
- # Interim assistant messages and thinking_progress are independent of tool progress (same
- # queue). Mattermost requires a per-platform opt-in: scratch text leaks into public threads.
- interim_assistant_messages_mode = _display_surface_mode(
- "interim_assistant_messages", default=True, require_platform_override_for={Platform.MATTERMOST},
- )
- interim_assistant_messages_enabled = not is_webhook and not bot_peer and interim_assistant_messages_mode != "off"
- _thinking_enabled = _display_surface_mode(
- "thinking_progress", default=False, require_platform_override_for={Platform.MATTERMOST},
- ) != "off" and not bot_peer
- # Slack-native task cards (#29483): when the Slack adapter's opt-in is set, tool progress renders as
- # native plan/task cards via chat.startStream — the progress queue is needed even though Slack keeps
- # ordinary text tool_progress off by default (requiring both flags would silently leave the native
- # feature inactive).
- # Cards are still tool progress. Slack's TIER default (``off``) only quiets the text lane so
- # cards stay on for unconfigured installs, but an operator who WRITES ``tool_progress: off``
- # (global, platform override, or legacy overrides) has asked for no tool progress at all and
- # gets no cards either. Every other explicit mode keeps the card lane.
- _native_slack_task_cards = False
- if (
- source.platform == Platform.SLACK
- and hasattr(adapter, "native_task_cards_enabled")
- and not (_tool_progress_explicit and progress_mode == "off") and not bot_peer
- ):
- try:
- _native_slack_task_cards = bool(adapter.native_task_cards_enabled())
- except Exception:
- logger.debug("Slack native task-card config check failed", exc_info=True)
- return self._RunAgentDisplay(
- user_config=user_config, platform_key=platform_key, enabled_toolsets=enabled_toolsets,
- disabled_toolsets=disabled_toolsets, resolve_display_setting=resolve_display_setting,
- progress_mode=progress_mode, progress_grouping=progress_grouping,
- _display_surface_mode=_display_surface_mode,
- tool_progress_enabled=tool_progress_enabled, _live_status_mode=_live_status_mode,
- _live_status_adapter=_live_status_adapter, log_mode_enabled=log_mode_enabled,
- log_queue=queue.Queue() if log_mode_enabled else None,
- interim_assistant_messages_enabled=interim_assistant_messages_enabled,
- _thinking_enabled=_thinking_enabled, _native_slack_task_cards=_native_slack_task_cards,
- needs_progress_queue=tool_progress_enabled or _thinking_enabled or _native_slack_task_cards,
- _generic_status_phrase=_generic_status_phrase,
- )
- # _RunAgentDisplay fields copied verbatim onto the TurnContext.
- _DISPLAY_TO_TURN_CTX = (
- "_live_status_adapter", "_live_status_mode", "_thinking_enabled", "progress_mode",
- "progress_grouping", "tool_progress_enabled", "log_queue", "resolve_display_setting",
- "user_config", "enabled_toolsets", "disabled_toolsets", "log_mode_enabled",
- "interim_assistant_messages_enabled", "needs_progress_queue", "_native_slack_task_cards",
- )
- def _run_agent_build_turn_context(
- self, disp: GatewayRunner._RunAgentDisplay, AIAgent: Any, *, message: str, source: SessionSource,
- session_key: Optional[str], run_generation: Optional[int], **turn_params,
- ) -> tuple[TurnContext, TurnRunner, Any]:
- """Build the ``TurnContext`` and its ``TurnRunner``; ``turn_params`` (history, context_prompt,
- session_id, persist_user_*, …) are stored verbatim. Returns ``(turn_ctx, turn_runner,
- cleanup_adapter)``."""
- from gateway.run_turn_runner import TurnRunner
- # Discord voice "verbal ack" on the FIRST tool call (discord.voice_fx.enabled): resolve the
- # guild whose voice connection is bound to this text channel (mirrors DiscordAdapter.play_tts).
- _voice_ack_guild: list[Optional[int]] = [self._voice_ack_guild_for(source)]
- # Auto-cleanup of temporary progress bubbles needs a real ``delete_message`` (getattr on the
- # type: a fake adapter without it means "can't delete", not a crash).
- _cleanup_progress = bool(
- disp.resolve_display_setting(disp.user_config, disp.platform_key, "cleanup_progress")
- )
- _cleanup_adapter = self._delivery_adapter_for(source) if _cleanup_progress else None
- if _cleanup_adapter is not None and getattr(type(_cleanup_adapter), "delete_message", None) in (
- None, BasePlatformAdapter.delete_message,
- ):
- _cleanup_progress = False
- _cleanup_adapter = None
- # The one-slot progress/holder containers shared with the callbacks are TurnContext defaults.
- turn_ctx = TurnContext(
- source=source, message=message, AIAgent=AIAgent, session_key=session_key,
- run_generation=run_generation, _cleanup_progress=_cleanup_progress,
- _run_still_current=self._run_still_current_fn(session_key, run_generation),
- progress_queue=queue.Queue() if disp.needs_progress_queue else None,
- _voice_ack_guild=_voice_ack_guild, _voice_ack_loop=asyncio.get_running_loop(),
- **{name: getattr(disp, name) for name in self._DISPLAY_TO_TURN_CTX}, **turn_params,
- )
- turn_runner = TurnRunner(self, turn_ctx)
- turn_ctx.mute_notification_reply = diagnostic_turn_muted(
- turn_ctx.persist_user_display_metadata, source.platform, turn_ctx.user_config)
- # Agent tool-lifecycle callbacks live on the runner (bound methods, same signatures).
- turn_ctx.progress_callback = turn_runner.progress_callback
- turn_ctx.voice_ack_callback = turn_runner.voice_ack_callback
- turn_ctx.native_tool_start_callback = turn_runner.combined_tool_start_callback
- turn_ctx.native_tool_complete_callback = turn_runner.native_tool_complete_callback
- return turn_ctx, turn_runner, _cleanup_adapter
- def _thread_metadata_for_progress(
- self, source: SessionSource, event_message_id: Optional[str], _progress_thread_id: Any,
- _relay_prospective_thread_id: Optional[str],
- ) -> Optional[dict[str, Any]]:
- """Thread metadata for a progress-lane send; relay Discord auto-thread lane falls back to the reply anchor.
- The connector will auto-thread on the reply anchor (thread is born on its FIRST send), so
- carrying it routes progress / status bubbles into the same thread as the final reply."""
- if not _progress_thread_id:
- metadata = None
- elif _progress_thread_id == source.thread_id:
- metadata = self._thread_metadata_for_source(source, event_message_id)
- else:
- metadata = self._thread_metadata_for_target(
- source.platform, source.chat_id, _progress_thread_id,
- chat_type=getattr(source, "chat_type", None), reply_to_message_id=event_message_id,
- )
- if metadata is None and _relay_prospective_thread_id:
- metadata = {"reply_to_message_id": event_message_id}
- return metadata
- def _run_agent_progress_threading(
- self, source: SessionSource, event_message_id: Optional[str], _native_slack_task_cards: bool
- ) -> tuple[Optional[dict], Optional[str], Optional[dict]]:
- """Resolve where progress bubbles are threaded (platform-specific).
- Returns ``(progress_metadata, progress_reply_to, status_thread_metadata)``; the latter is
- for status / approval / stream sends (Feishu topics need the triggering message id via the
- reply API, so carry it as a fallback). Slack and Buzz honour the user's reply_in_thread
- opt-out: never synthesise a thread for progress, or every later reply inherits it."""
- from gateway.run import _non_conversational_metadata, _resolve_progress_thread_id
- is_buzz = str(getattr(source.platform, "value", source.platform) or "").lower() == "buzz"
- _progress_reply_in_thread = True
- _adapter = self._delivery_adapter_for(source) if source.platform == Platform.SLACK or is_buzz else None
- if _adapter is not None:
- try:
- if is_buzz:
- _progress_reply_in_thread = getattr(_adapter, "_reply_to_mode", "first") != "off"
- else:
- # Relay lane: the adapter owns mode resolution; native lane: flat extra key.
- _mode_fn = getattr(_adapter, "_effective_reply_in_thread", None)
- _progress_reply_in_thread = bool(
- _mode_fn() if callable(_mode_fn) else _adapter.config.extra.get("reply_in_thread", True)
- )
- except Exception:
- _progress_reply_in_thread = True
- _progress_thread_id = _resolve_progress_thread_id(
- source.platform, source.thread_id, event_message_id, reply_in_thread=_progress_reply_in_thread,
- )
- # Relay Discord auto-thread lane: the connector stamps prospective_thread_id at ingest.
- _relay_prospective_thread_id = (
- str(getattr(source, "prospective_thread_id", None))
- if source.platform == Platform.DISCORD
- and getattr(source, "delivered_via_upstream_relay", False)
- and getattr(source, "prospective_thread_id", None)
- and not source.thread_id
- else None
- )
- _progress_metadata = _non_conversational_metadata(
- self._thread_metadata_for_progress(
- source, event_message_id, _progress_thread_id, _relay_prospective_thread_id,
- ),
- platform=source.platform,
- )
- if _native_slack_task_cards:
- # chat.startStream in channels requires the recipient team/user pair; harmless elsewhere.
- _progress_metadata = dict(_progress_metadata or {})
- if source.scope_id:
- _progress_metadata.setdefault("recipient_team_id", source.scope_id)
- _progress_metadata.setdefault("slack_team_id", source.scope_id)
- if source.user_id:
- _progress_metadata.setdefault("recipient_user_id", source.user_id)
- # Buzz has no native thread_id: thread via reply-to unless the user opted out.
- _progress_reply_to = (
- event_message_id
- if (source.platform in (Platform.FEISHU, Platform.MATTERMOST) and source.thread_id and event_message_id)
- or (is_buzz and event_message_id and _progress_reply_in_thread)
- or _relay_prospective_thread_id
- else None
- )
- if source.platform == Platform.FEISHU and source.thread_id and event_message_id:
- _status_thread_metadata = {"thread_id": _progress_thread_id, "reply_to_message_id": event_message_id}
- else:
- _status_thread_metadata = self._thread_metadata_for_progress(
- source, event_message_id, _progress_thread_id, _relay_prospective_thread_id,
- )
- return _progress_metadata, _progress_reply_to, _status_thread_metadata
- async def _run_agent_write_tool_log(self, log_queue: Any) -> None:
- """Drain log_queue and append tool-call lines to tool_calls.log (tool_progress=log).
- RotatingFileHandler (5MB × 3) bounds the log; RedactingFormatter keeps secrets off disk."""
- if log_queue is None:
- return
- tool_logger = _tool_call_logger()
- try:
- while True:
- try:
- tool_logger.info("%s", log_queue.get_nowait())
- except queue.Empty:
- await asyncio.sleep(0.3)
- except Exception as e:
- logger.error("write_tool_log error: %s", e)
- await asyncio.sleep(1)
- except asyncio.CancelledError:
- pass
- finally:
- # Drain remaining entries so late tool calls from the final iteration aren't lost.
- with suppress(Exception):
- while True:
- tool_logger.info("%s", log_queue.get_nowait())
- with suppress(Exception):
- for handler in tool_logger.handlers:
- handler.flush()
- def _run_agent_start_streaming_tts(
- self, source: SessionSource, message_type: Optional[str],
- _status_thread_metadata: Optional[dict[str, Any]], streaming_tts_consumer_holder: list,
- ) -> None:
- """Start the streaming-TTS consumer for a voice-input turn on an auto-TTS chat.
- Created on the gateway loop thread (not run_sync's executor); an inactive consumer leaves
- the holder None so the whole-file fallback path runs."""
- # Skip when streaming TTS already delivered audio for this turn (#60671).
- # This avoids a cross-scope NameError: the outer interrupt / finalisation paths reference the
- # consumer via ``streaming_tts_consumer_holder[0]``. Gates: voice input, auto-TTS enabled for this
- # chat, adapter supports streaming, and a usable streaming TTS provider configured. See #60671.
- _stts_adapter = self._delivery_adapter_for(source)
- _is_voice_input = (
- message_type is not None
- and str(getattr(message_type, "value", message_type)).lower() == "voice"
- )
- if _stts_adapter is None or not _is_voice_input or not _stts_adapter._should_auto_tts_for_chat(source.chat_id):
- return
- try:
- from gateway.streaming_tts_consumer import StreamingTTSConsumer
- from tools.tts_tool import _load_tts_config
- _stts_consumer = StreamingTTSConsumer(
- adapter=_stts_adapter, chat_id=source.chat_id, tts_config=_load_tts_config(),
- loop=self._gateway_loop or asyncio.get_event_loop(),
- metadata=_status_thread_metadata,
- )
- if _stts_consumer.active:
- streaming_tts_consumer_holder[0] = _stts_consumer
- _stts_consumer.start()
- except Exception as _stts_err:
- logger.debug("Could not set up streaming TTS consumer: %s", _stts_err)
- async def _run_agent_stream_consumer_task(self, stream_consumer_holder: list) -> None:
- """Wait (up to 10s) for the stream consumer to be created inside run_sync, then run it."""
- for _ in range(200):
- if stream_consumer_holder[0] is not None:
- await stream_consumer_holder[0].run()
- return
- await asyncio.sleep(0.05)
- @staticmethod
- async def _await_stream_task(stream_task) -> None:
- """Give the stream consumer task 5s to flush, then cancel it."""
- try:
- await asyncio.wait_for(stream_task, timeout=5.0)
- except (TimeoutError, asyncio.CancelledError):
- stream_task.cancel()
- with suppress(asyncio.CancelledError):
- await stream_task
- async def _run_agent_track_agent(self, turn_ctx: TurnContext) -> None:
- """Track this agent as running for the session (interrupt support) once it is created — only
- if this run is still current, else leave the newer run's slot alone."""
- session_key, run_generation, agent_holder = turn_ctx.session_key, turn_ctx.run_generation, turn_ctx.agent_holder
- while agent_holder[0] is None:
- await asyncio.sleep(0.05)
- if not session_key:
- return
- if run_generation is not None and not self._is_session_run_current(session_key, run_generation):
- logger.info(
- "Skipping stale agent promotion for %s — generation %s is no longer current",
- session_key or "", run_generation,
- )
- return
- turn_state = self._session_state(session_key).turn
- turn_state.agent, turn_state.ctx = agent_holder[0], turn_ctx
- if self._draining:
- self._update_runtime_status("draining")
- async def _run_agent_fire_pending_interrupt(
- self, adapter: Any, agent: Any, source: SessionSource, session_key: str,
- _interrupt_detected: asyncio.Event, streaming_tts_consumer_holder: list, *,
- log_context: str, log: Callable[[], None],
- ) -> None:
- """Peek the adapter's pending event, transcribe voice, then signal the agent + abort streaming TTS.
- Peek WITHOUT consuming: the event must stay for the post-run ``_dequeue_pending_event()``
- (popping races the agent finishing). Transcribe BEFORE signaling so voice interrupts carry
- the real transcript."""
- from gateway.run import _build_media_placeholder
- _peek_event = adapter._pending_messages.get(session_key)
- pending_text = None
- if _peek_event is not None:
- pending_text = _peek_event.text or ""
- if self._pending_event_audio_paths(_peek_event):
- pending_text, _ = await self._transcribe_and_echo_pending_voice(
- _peek_event, adapter, source, pending_text, log_context=log_context,
- metadata={"thread_id": source.thread_id} if source.thread_id else None,
- )
- elif not pending_text and (getattr(_peek_event, "media_urls", None) or []):
- pending_text = _build_media_placeholder(_peek_event)
- log()
- agent.interrupt(pending_text)
- _interrupt_detected.set()
- # Abort streaming TTS on barge-in.
- # See #60671.
- # See #60671.
- # See #60671.
- # Finalize the streaming-TTS consumer (#60671). finish() is called from the outer event-loop thread
- # (not the executor worker) so early returns from run_sync are also finalised. wait_complete()
- # drains queued audio; on timeout the consumer is aborted unconditionally — if audio was audible,
- # suppression is preserved so the gateway does not replay from the beginning; if no audio was
- # audible, the whole-file fallback path is permitted.
- _stts = streaming_tts_consumer_holder[0]
- if _stts is not None:
- _stts.abort("barge-in")
- async def _run_agent_monitor_for_interrupt(self, turn_ctx: TurnContext, _interrupt_detected: asyncio.Event) -> None:
- """Poll the adapter for interrupts (new messages) every 200ms and signal the agent.
- Level 1 (base.py) catches regular text before _handle_message(); the inactivity poll loop
- has a BACKUP check in case this task dies. Keyed by session_key, NOT source.chat_id."""
- source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder
- streaming_tts_consumer_holder = turn_ctx.streaming_tts_consumer_holder
- if not session_key:
- return
- while True:
- await asyncio.sleep(0.2)
- try:
- # Re-resolve the adapter each iteration so reconnects don't leave a stale reference.
- _adapter = self._delivery_adapter_for(source)
- if not _adapter:
- continue
- if hasattr(_adapter, 'has_pending_interrupt') and _adapter.has_pending_interrupt(session_key):
- agent = agent_holder[0]
- if agent:
- await self._run_agent_fire_pending_interrupt(
- _adapter, agent, source, session_key, _interrupt_detected,
- streaming_tts_consumer_holder, log_context="Voice-interrupt",
- log=lambda: logger.debug("Interrupt detected from adapter, signaling agent..."),
- )
- break
- except asyncio.CancelledError:
- raise
- except Exception as _mon_err:
- logger.debug("monitor_for_interrupt error (will retry): %s", _mon_err)
- async def _run_agent_backup_interrupt_check(
- self, turn_ctx: TurnContext, _interrupt_detected: asyncio.Event, interrupt_monitor: asyncio.Task,
- ) -> None:
- """Backup interrupt check: if the monitor task died or missed the interrupt, catch it here."""
- source, session_key = turn_ctx.source, turn_ctx.session_key
- if _interrupt_detected.is_set() or not session_key:
- return
- _backup_adapter = self._delivery_adapter_for(source)
- _backup_agent = turn_ctx.agent_holder[0]
- if (_backup_adapter and _backup_agent
- and hasattr(_backup_adapter, 'has_pending_interrupt')
- and _backup_adapter.has_pending_interrupt(session_key)):
- await self._run_agent_fire_pending_interrupt(
- _backup_adapter, _backup_agent, source, session_key, _interrupt_detected,
- turn_ctx.streaming_tts_consumer_holder,
- log_context="Voice-backup-interrupt",
- log=lambda: logger.info(
- "Backup interrupt detected for session %s (monitor task state: %s)",
- session_key, "done" if interrupt_monitor.done() else "running",
- ),
- )
- @staticmethod
- def _run_agent_stream_confirmed_final_delivery(consumer, final_text: str, *, previewed: bool = False) -> bool:
- """True only when the actual final reply reached the user: a finalize call may carry only the
- last preview snapshot, so reconcile against the recorded payload — a demonstrable mismatch
- (False) overrides the flag; None keeps legacy trust."""
- if consumer is None:
- return False
- if getattr(consumer, "final_response_sent", False):
- matcher = getattr(consumer, "delivered_final_matches", None)
- if callable(matcher):
- with suppress(Exception):
- if matcher(final_text) is False:
- return False
- return True
- # Exact-text match against what the consumer DURABLY delivered (commentary, segments, and the
- # visible prefix only once a real send landed) — safe without the ``previewed`` flag. The codex
- # app-server bridge delivers the final agentMessage through the commentary path and never sets
- # response_previewed (#74248 / #80519); gating on the flag re-sent every such reply. Mismatching
- # commentary still returns False, so a distinct final answer is never suppressed (#65919). Draft
- # frames are ephemeral and must not count: after draft streaming + a failed finalize send this
- # predicate must stay False so the fallback final send still fires (#51828 / #33793).
- has_delivered_text = getattr(consumer, "has_durably_delivered_text", None)
- if callable(has_delivered_text):
- try:
- return bool(has_delivered_text(final_text))
- except Exception:
- return False
- return False
- def _run_agent_start_turn_worker(self, turn_ctx: TurnContext, run_sync: Callable[[], Any]) -> GatewayRunner._RunAgentWorker:
- """Schedule ``run_sync`` on the executor plus the inactivity watchdog thread.
- *Inactivity* timeout (agent.gateway_timeout / HERMES_AGENT_TIMEOUT, env wins; 0 = unlimited),
- not wall-clock. The daemon watchdog is independent of asyncio: cgroup memory reclaim can
- starve the loop that runs the normal timeout poll."""
- from gateway.run import _float_env, _watch_gateway_turn_inactivity
- from tools.process_registry import process_registry
- agent_holder, session_key, run_generation = turn_ctx.agent_holder, turn_ctx.session_key, turn_ctx.run_generation
- _agent_timeout, _agent_warning = (
- v if v > 0 else None
- for v in (_float_env("HERMES_AGENT_TIMEOUT", 1800), _float_env("HERMES_AGENT_TIMEOUT_WARNING", 900))
- )
- # background=true processes survive a turn: reap only children created by THIS turn on timeout.
- _turn_task_id = turn_ctx.session_id or ""
- # The daemon watchdog is independent of asyncio: cgroup memory reclaim may starve the event loop
- # that runs the normal timeout poll, but it need not also postpone cleanup until the loop recovers
- # (#76115).
- _turn_process_baseline = process_registry.snapshot_running_ids(_turn_task_id)
- turn_ctx.process_task_id = _turn_task_id
- turn_ctx.process_baseline = _turn_process_baseline
- # task_id is session-scoped: gate the reap on this claim still being current so a replacement
- # turn's fresh process isn't killed by this turn's stale baseline.
- worker = self._RunAgentWorker(
- agent_timeout=_agent_timeout, agent_warning=_agent_warning, task_id=_turn_task_id,
- process_baseline=_turn_process_baseline, worker_done=threading.Event(),
- timeout_fired=threading.Event(), cleanup_lock=threading.Lock(),
- is_current=(
- (lambda: self._is_session_run_current(session_key, run_generation))
- if run_generation is not None
- else (lambda: True)
- ),
- )
- def _run_sync_with_timeout_lifecycle():
- try:
- return run_sync()
- finally:
- worker.worker_done.set()
- # `.turn.agent` stays reachable until the *next* turn is claimed; clearing the
- # ownership markers now means a /stop on the finished turn no longer reaps background
- # work it left running.
- # `.turn.agent` on the session state is only reset to _AGENT_PENDING_SENTINEL when the
- # *next* turn is claimed (see _session_state(...).turn.agent = ... at claim time), so a
- # stale reference to this exact agent instance stays reachable from
- # _interrupt_and_clear_session() until then. See #76115.
- _finished_agent = agent_holder[0] if agent_holder else None
- if _finished_agent is not None:
- _finished_agent._gateway_turn_process_task_id = ""
- _finished_agent._gateway_turn_process_baseline = frozenset()
- if _agent_timeout is not None:
- threading.Thread(
- target=_watch_gateway_turn_inactivity,
- kwargs={
- "agent_holder": agent_holder, "timeout": _agent_timeout, "poll_interval": 5.0,
- **self._reaper_kwargs(worker),
- },
- name=f"gateway-turn-watchdog-{_turn_task_id[:12]}",
- daemon=True,
- ).start()
- from gateway.session_runtime_workers import start_turn_worker
- start_turn_worker(self, worker, agent_holder, _run_sync_with_timeout_lifecycle)
- return worker
- @staticmethod
- def _reaper_kwargs(worker: GatewayRunner._RunAgentWorker) -> dict:
- """Shared kwargs of the watchdog + timeout-reaper threads."""
- return {
- **{k: getattr(worker, k) for k in ("task_id", "process_baseline", "worker_done", "timeout_fired", "cleanup_lock")},
- "is_still_current": worker.is_current,
- }
- @staticmethod
- def _agent_activity_summary(agent: Any) -> dict:
- """``agent.get_activity_summary()`` or ``{}`` when unavailable / failing."""
- if agent and hasattr(agent, "get_activity_summary"):
- with suppress(Exception):
- return agent.get_activity_summary()
- return {}
- async def _run_agent_inactivity_warning(self, worker, source, _status_thread_metadata) -> None:
- """Staged one-shot warning before the inactivity timeout escalates."""
- from gateway.run import _interim_metadata
- _warn_adapter = self._delivery_adapter_for(source) if automatic_notices_allowed(source) else None
- if not _warn_adapter:
- return
- try:
- await _warn_adapter.emit_warning(
- source.chat_id, t(
- "gateway.progress.stuck_warning",
- minutes=int(worker.agent_warning // 60) or 1,
- remaining=int((worker.agent_timeout - worker.agent_warning) // 60) or 1),
- metadata=_interim_metadata(_status_thread_metadata), logical_platform=source.platform,
- )
- except Exception as _warn_err:
- logger.debug("Inactivity warning send error: %s", _warn_err)
- def _run_agent_timeout_result(self, worker, turn_ctx: TurnContext) -> dict:
- """Synthetic failed run dict for an inactivity timeout, with the activity-tracker diagnostic;
- interrupts the agent if it is still running so the thread pool worker is freed."""
- from gateway.run import _INTERRUPT_REASON_TIMEOUT, _INTERRUPT_TOOL_REASON_TIMEOUT, request_hard_interrupt
- from gateway.session_results import record_terminal_failure
- session_key, result_holder, tools_holder = turn_ctx.session_key, turn_ctx.result_holder, turn_ctx.tools_holder
- _timed_out_agent = turn_ctx.agent_holder[0]
- _activity = self._agent_activity_summary(_timed_out_agent)
- _last_desc = _activity.get("last_activity_desc", "unknown")
- _secs_ago = _activity.get("seconds_since_activity", 0)
- _cur_tool = _activity.get("current_tool")
- _iter_n = _activity.get("api_call_count", 0)
- _iter_max = _activity.get("max_iterations", 0)
- # Operator-facing log keeps the raw resolved value; only the user-facing lines hide the sentinel.
- logger.error(
- "Agent idle for %.0fs (timeout %.0fs) in session %s "
- "| last_activity=%s | iteration=%s/%s | tool=%s",
- _secs_ago, worker.agent_timeout, session_key, _last_desc, _iter_n, _iter_max,
- _cur_tool or "none",
- )
- if _timed_out_agent:
- request_hard_interrupt(_timed_out_agent, _INTERRUPT_REASON_TIMEOUT, tool_reason=_INTERRUPT_TOOL_REASON_TIMEOUT)
- _timeout_mins = int(worker.agent_timeout // 60) or 1
- _iter_progress = format_iteration_progress(_iter_n, _iter_max)
- _diag_lines = [t("gateway.progress.timeout_header", minutes=_timeout_mins)]
- if _cur_tool:
- _diag_lines.append(t(
- "gateway.progress.timeout_stuck_tool",
- tool=_cur_tool, seconds=f"{_secs_ago:.0f}", progress=_iter_progress))
- else:
- _diag_lines.append(t(
- "gateway.progress.timeout_last_activity",
- activity=_last_desc, seconds=f"{_secs_ago:.0f}", progress=_iter_progress))
- _diag_lines.append(t("gateway.progress.timeout_hint"))
- return record_terminal_failure({
- "final_response": "\n".join(_diag_lines),
- "messages": result_holder[0].get("messages", []) if result_holder[0] else [],
- "api_calls": _iter_n,
- "tools": tools_holder[0] or [],
- "history_offset": 0,
- "failed": True,
- })
- async def _run_agent_await_turn_worker(
- self, worker: GatewayRunner._RunAgentWorker, turn_ctx: TurnContext,
- _interrupt_detected: asyncio.Event, interrupt_monitor: asyncio.Task,
- ) -> Any:
- """Poll the executor future (inactivity timeout + backup interrupt checks); return its result,
- or a synthetic failed run dict on inactivity timeout. Polls even with an unlimited timeout
- so the backup interrupt check runs if monitor_for_interrupt() silently died."""
- from gateway.run import _abandon_timed_out_gateway_turn
- agent_holder = turn_ctx.agent_holder
- _warning_fired = False
- while True:
- done, _ = await asyncio.wait({worker.executor_task}, timeout=5.0)
- if done:
- # Prefer the real result even if the watchdog fired in the same window (the run already
- # persisted its reply).
- return worker.executor_task.result()
- if worker.agent_timeout is not None:
- if worker.timeout_fired.is_set():
- break
- _idle_secs = turn_ctx.awake_idle.measure(self._agent_activity_summary(agent_holder[0]).get("seconds_since_activity", 0.0))
- if not _warning_fired and worker.agent_warning is not None and _idle_secs >= worker.agent_warning:
- _warning_fired = True
- await self._run_agent_inactivity_warning(worker, turn_ctx.source, turn_ctx._status_thread_metadata)
- if _idle_secs >= worker.agent_timeout:
- threading.Thread(
- target=_abandon_timed_out_gateway_turn,
- kwargs={"agent_holder": agent_holder, **self._reaper_kwargs(worker)},
- name=f"gateway-turn-reaper-{worker.task_id[:12]}", daemon=True,
- ).start()
- break
- await self._run_agent_backup_interrupt_check(turn_ctx, _interrupt_detected, interrupt_monitor)
- return self._run_agent_timeout_result(worker, turn_ctx)
- def _run_agent_evict_on_fallback(self, turn_ctx: TurnContext) -> None:
- """Evict the cached agent when a fallback model activated on a SUCCESSFUL run (so /model shows
- the active model and the next message retries the primary). Skip failed runs: evicting
- would loop bad model → fallback → evict → recreate."""
- session_key, _agent, _result_for_fb = turn_ctx.session_key, turn_ctx.agent_holder[0], turn_ctx.result_holder[0]
- if _agent is None or not hasattr(_agent, 'model') or (_result_for_fb and _result_for_fb.get("failed")):
- return
- # A provider fallback is drift even when it serves the configured model name on another endpoint.
- if getattr(_agent, "_provider_fallback_active", False) is not True:
- # A LOCAL session runs its frozen launch policy's model, not the gateway's configured one.
- from gateway.session_authorities import active_authority
- from gateway.session_policy import policy_for_source
- policy = policy_for_source(self, turn_ctx.source)
- _cfg_model = self._fallback_baseline_model(
- session_key, turn_ctx.source, _agent, policy_model=policy.current_model(active_authority(self)) if policy and policy.model else None)
- if _agent.model == _cfg_model or self._is_intentional_model_switch(session_key, _agent, _cfg_model):
- return
- _agent._evicted_on_fallback = session_key or True
- self._evict_cached_agent(session_key)
- async def _run_agent_finalize_streaming_tts(self, turn_ctx: TurnContext, adapter: Any) -> None:
- """Finalize the streaming-TTS consumer on the outer event-loop thread (covers early returns
- from run_sync). On drain timeout abort to free the task — audible streams keep whole-file
- suppression, silent streams stay eligible for the whole-file fallback."""
- _stts = turn_ctx.streaming_tts_consumer_holder[0]
- if _stts is None:
- return
- _stts.finish()
- try:
- await _stts.wait_complete(timeout=10.0)
- except Exception as _stts_done_err:
- logger.debug("streaming TTS wait_complete error: %s", _stts_done_err)
- if not _stts.done:
- _stts.abort("streaming TTS finalisation timeout")
- await _stts.wait_complete(timeout=2.0)
- if _stts.suppress_whole_file and adapter is not None:
- _mark_turn = getattr(adapter, "_mark_streaming_tts_completed_turn", None)
- if callable(_mark_turn):
- _mark_turn(turn_ctx.session_key, turn_ctx.run_generation)
- async def _run_agent_drain_pending(
- self, result: Any, adapter: Any, source: SessionSource, session_key: Optional[str]
- ) -> tuple[Any, Optional[str]]:
- """Dequeue the adapter's pending / interrupt / leftover-steer follow-up as ``(pending_event, pending)``.
- Keyed by session_key (not source.chat_id) to match the adapter's storage keys."""
- from gateway.run import _build_media_placeholder, _dequeue_pending_event, _is_control_interrupt_message
- from gateway.session_ingress import executing_admission
- pending_event = None
- pending = None
- if result and adapter and session_key:
- pending_event = _dequeue_pending_event(adapter, session_key)
- if result.get("compression_exhausted") and pending_event is not None:
- self._defer_past_exhaustion_reset(result, adapter, source, session_key, pending_event)
- return None, None
- # /queue overflow: promote the next queued event into the consumed "next-up" slot so the
- # recursive drain sees it (keeps FIFO order; a mid-chain /queue can't jump the queue).
- pending_event = self._promote_queued_event(session_key, adapter, pending_event)
- if result.get("interrupted") and not pending_event and result.get("interrupt_message"):
- interrupt_message = result.get("interrupt_message")
- if _is_control_interrupt_message(interrupt_message):
- logger.info(
- "Ignoring control interrupt message for session %s: %s",
- session_key or "?", interrupt_message,
- )
- else:
- # FIFO already has this send.
- pending = None if executing_admission.get() else interrupt_message
- elif pending_event:
- # Transcribe audio BEFORE it becomes the next user turn (real transcript, not a path).
- _pending_text = pending_event.text or ""
- if self._pending_event_audio_paths(pending_event):
- pending, _ = await self._transcribe_and_echo_pending_voice(
- pending_event, adapter, source, _pending_text, log_context="Voice-drain",
- metadata={"thread_id": source.thread_id} if source.thread_id else None,
- )
- pending = pending or _build_media_placeholder(pending_event)
- else:
- pending = _pending_text or _build_media_placeholder(pending_event)
- if pending:
- logger.debug("Processing queued message after agent completion: '%s...'", pending[:40])
- # Leftover /steer (arrived after the last tool batch): deliver as the next user turn.
- pending_event, pending = self._merge_leftover_steer(result, adapter, source, session_key, pending_event, pending)
- # Safety net: a pending slash command is never passed to the agent as user input.
- if pending and pending.strip().startswith("/"):
- _pending_cmd_word = pending.strip().split(None, 1)[0][1:].lower()
- if _pending_cmd_word:
- with suppress(Exception):
- from hermes_cli.commands import resolve_command as _rc_pending
- if _rc_pending(_pending_cmd_word):
- logger.info(
- "Discarding command '/%s' from pending queue — "
- "commands must not be passed as agent input", _pending_cmd_word,
- )
- pending_event = None
- pending = None
- if self._draining and getattr(pending_event, "_admitted_before_drain", False) is not True:
- if pending_event or pending:
- logger.info("Discarding pending follow-up for session %s during gateway %s", session_key or "?", self._status_action_label())
- # Without a result the caller runs nothing next: leave the queue to the adapter.
- pending_event = self._take_wake_admitted_before_drain(session_key, adapter) if result and adapter and session_key else None
- pending = (pending_event.text or _build_media_placeholder(pending_event)) if pending_event is not None else None
- return pending_event, pending
- async def _run_agent_deliver_first_response(
- self, turn_ctx: TurnContext, adapter: Any, response: Any, result: Any, stream_task: Any,
- ) -> None:
- """Deliver the first response before a queued follow-up runs, unless streaming already did."""
- if turn_ctx.mute_notification_reply:
- return
- session_key = turn_ctx.session_key
- _sc = turn_ctx.stream_consumer_holder[0]
- if _sc and stream_task:
- try:
- await self._await_stream_task(stream_task)
- except Exception as e:
- logger.debug("Stream consumer wait before queued message failed: %s", e)
- # Delivery uses the finalized task result (empty/failure normalization), not raw ``result``.
- _delivery_result = response if isinstance(response, dict) else (result or {})
- first_response = _delivery_result.get("final_response", "")
- _already_streamed = self._run_agent_stream_confirmed_final_delivery(
- _sc, first_response, previewed=bool(_delivery_result.get("response_previewed")),
- )
- # Same silence predicate as the normal path, else this branch leaks the literal marker.
- _silent = False
- if self._is_intentional_silence(_delivery_result, first_response):
- _silent = self._silence_allowed_for_source(
- turn_ctx.source, turn_ctx.persist_user_display_kind, turn_ctx.reply_expected, _delivery_result)
- if _silent:
- logger.info(
- "Queued follow-up for session %s: suppressing intentional silence marker before continuing.",
- session_key or "?",
- )
- first_response = ""
- else:
- logger.warning(
- "Queued follow-up for session %s: replacing a human-turn silence marker.",
- session_key or "?",
- )
- first_response = _unexpected_silence_reply()
- _already_streamed = False
- await self._record_room_silence_decision(
- session_key, _delivery_result, _silent,
- display_kind=turn_ctx.persist_user_display_kind, reply_expected=turn_ctx.reply_expected)
- # Failed turns deliver their text but never their attachments (completed-turn parity).
- _deliver_media = not _delivery_result.get("failed")
- if first_response:
- logger.info(
- "Queued follow-up for session %s: final text delivery confirmed; delivering explicit media before continuing."
- if _already_streamed else
- "Queued follow-up for session %s: final stream delivery not confirmed; sending first response before continuing.",
- session_key or "?",
- )
- try:
- _text_delivered = await self._deliver_queued_first_response(
- first_response, source=turn_ctx.source, adapter=adapter,
- metadata=turn_ctx._status_thread_metadata, event_message_id=turn_ctx.event_message_id,
- text_already_delivered=_already_streamed,
- deliver_media=_deliver_media, stream_consumer=_sc,
- # The text send records a delivery-ledger obligation under this key, keyed on
- # the raw inbound id (the anchor above is only the reply target).
- session_key=session_key, inbound_message_id=turn_ctx.inbound_message_id,
- )
- except Exception as e:
- logger.warning("Failed to send first response before queued message: %s", e)
- else:
- # One source of truth for "this turn's final already reached the chat": the normal
- # completion path (`_hmwa_deliver_turn_response`) consults ``already_sent`` on the
- # result the queued lane hands back. Every early `return result` after this point
- # (follow-up text refused, stale goal continuation) otherwise re-sends the text the
- # fallback just delivered — the #81052 duplicate. A REFUSED send reports False, and
- # the completion send stays the fallback so the user is not left with nothing.
- if _text_delivered and isinstance(result, dict):
- result["already_sent"] = True
- # The queued lane already uploaded this response's MEDIA: attachments; without
- # this the completion path's already_sent rescan uploads every file twice.
- result["media_already_delivered"] = _deliver_media
- # Release deferred bg-review notifications: pop (no double-fire in base.py's finally) and call.
- _bg_cb = self._pop_post_delivery_callback(adapter, session_key, turn_ctx.run_generation)
- if callable(_bg_cb):
- with suppress(Exception):
- _bg_result = _bg_cb()
- if inspect.isawaitable(_bg_result):
- await _bg_result
- async def _run_agent_queued_followup(
- self, turn_ctx: TurnContext, adapter: Any, pending: Optional[str], pending_event: Any,
- response: Any, result: Any, stream_task: Any,
- ) -> Any:
- """Run the queued / interrupting follow-up as the next turn (recursive ``_run_agent``)."""
- from gateway.run import _preserve_queued_followup_history_offset
- source, session_id, session_key, run_generation = (
- turn_ctx.source, turn_ctx.session_id, turn_ctx.session_key, turn_ctx.run_generation,
- )
- _interrupt_depth, history, _status_thread_metadata = (
- turn_ctx._interrupt_depth, turn_ctx.history, turn_ctx._status_thread_metadata,
- )
- logger.debug("Processing pending message: '%s...'", pending[:40])
- # Clear the interrupt event so the recursive _run_agent isn't re-interrupted (infinite loop).
- _active = getattr(adapter, "_active_sessions", None) if adapter else None
- if _active and session_key and session_key in _active:
- _active[session_key].clear()
- # Cap recursion depth (user keeps sending while the agent keeps failing).
- # (#816)
- if _interrupt_depth >= self._MAX_INTERRUPT_DEPTH:
- logger.warning(
- "Interrupt recursion depth %d reached for session %s — "
- "queueing message instead of recursing.", _interrupt_depth, session_key,
- )
- self._requeue_at_depth_cap(session_key, source, pending_event, pending)
- return turn_ctx.result_holder[0] or {"final_response": response, "messages": history}
- # Interrupted: discard the response ("Operation interrupted." is noise).
- if not result.get("interrupted"):
- await self._run_agent_deliver_first_response(turn_ctx, adapter, response, result, stream_task)
- updated_history = result.get("messages", history)
- next_source, next_message, next_session_key = source, pending, session_key
- # message_type is carried into the recursive call so queued voice turns can stream TTS.
- next_message_id = next_channel_prompt = next_message_type = None
- # The raw inbound id keys the delivery-ledger obligation for the follow-up's own final send,
- # distinct from the reply anchor above (None in forum topics). Carry it or two chained
- # topic turns with the same text would collide on one obligation id (queued-final-ledger).
- next_inbound_id = None
- # Queued Discord turns carry the same routing note as first turns; persist the authored text.
- next_persist_message = None
- next_display_kind = display_kind_for_event(pending_event)
- next_reply_expected = pending_event.reply_expected if pending_event is not None else None
- # See #60671.
- if pending_event is not None:
- next_source = getattr(pending_event, "source", None) or source
- if self._is_goal_continuation_event(pending_event) and not self._goal_still_active_for_session(session_id):
- logger.info(
- "Discarding stale goal continuation for session %s — goal is no longer active",
- session_key or "?",
- )
- return result
- # Resolve the follow-up's session key BEFORE preparing the inbound text: native image
- # paths are buffered under the key given and consumed under next_session_key.
- try:
- next_session_key = self._session_key_for_source(next_source)
- except Exception:
- logger.debug(
- "Queued follow-up session-key resolution failed; reusing %s",
- session_key or "?", exc_info=True,
- )
- next_message = await self._prepare_profile_scoped_inbound_message_text(
- event=pending_event, source=next_source, history=updated_history, session_key=next_session_key,
- )
- if next_message is None:
- return result
- from gateway.run_inbound import strip_discord_triggering_note
- next_persist_message = strip_discord_triggering_note(pending_event, next_message)
- next_message_id = self._reply_anchor_for_event(pending_event)
- next_inbound_id = str(pending_event.message_id) if getattr(pending_event, "message_id", None) else None
- next_channel_prompt, next_source = self._pinned_channel_inputs(
- next_session_key, pending_event.channel_prompt, next_source,
- internal=pending_event.internal or getattr(pending_event, "inherit_channel_pin", False),
- )
- if not pending_event.internal:
- # A drained human turn re-pins its channel inputs; make them durable like a first turn.
- await self._persist_prompt_pins(next_session_key, session_id)
- next_message_type = getattr(pending_event, "message_type", None)
- else:
- # Event-less interrupt/steer follow-ups continue the effective prompt
- # of the turn they are recursively following.
- next_channel_prompt = turn_ctx.channel_prompt
- # Clear the prior turn's streaming-TTS completion marker so the recursive turn isn't suppressed (#60671).
- _clear_adapter = self._delivery_adapter_for(source)
- _completed_turns = getattr(_clear_adapter, "_streaming_tts_completed_turns", None)
- _prior_key = getattr(_clear_adapter, "_streaming_tts_turn_key", None)
- if _completed_turns is not None and callable(_prior_key) and session_key and run_generation is not None:
- _pk = _prior_key(session_key, run_generation)
- if _pk:
- _completed_turns.discard(_pk)
- # Restart the typing indicator; the outer typing task may be stale.
- if _clear_adapter:
- with suppress(Exception):
- await _clear_adapter.send_typing(source.chat_id, metadata=typing_metadata_for_turn(
- _clear_adapter, source.chat_id, next_message_id, _status_thread_metadata))
- # Re-baseline the cached agent's message_count snapshot before recursing into the in-band queued
- # (/queue) follow-up turn. The first turn has completed and flushed its own user + assistant rows to
- # the SessionDB, so the cross-process coherence guard (#45966) — which this recursive _run_agent
- # call re-enters — would otherwise see the grown on-disk count against the stale build-time snapshot
- # and rebuild the agent on THIS process's OWN writes, destroying the prompt-cache prefix #46237 was
- # merged to preserve. The existing re-baseline in _handle_message_with_agent only runs after the
- # whole _run_agent chain unwinds — too late for the in-band follow-up. Use the same (session_key,
- # session_id) the recursive call runs under so the snapshot matches exactly what the follow-up's
- # guard will consult. Fail-safe in helper.
- # Acknowledge the follow-up the way an idle-session message is: this in-band drain is the only
- # place a queued/interrupting message ever runs, so base.py's hook site is never entered for it.
- # Resolve the adapter from the follow-up's OWN source — a multiplexed gateway can route it to a
- # different profile's adapter, and only that instance holds the per-message reaction state.
- from gateway.run_turn_followup_ack import _followup_cancel_outcome, _run_followup_processing_hook
- _hook_adapter = self._intake_adapter_for(next_source) if pending_event is not None else None
- await _run_followup_processing_hook(_hook_adapter, pending_event, "on_processing_start")
- # The re-baseline sits inside the try: a /stop landing on its DB await must still close the marker
- # (the helper's own ``except Exception`` does not catch cancellation).
- try:
- await self._refresh_agent_cache_message_count(session_key, session_id)
- followup_result = await self._run_agent(
- message=next_message, context_prompt=turn_ctx.context_prompt, history=updated_history,
- source=next_source, session_id=session_id, session_key=next_session_key,
- run_generation=run_generation, _interrupt_depth=_interrupt_depth + 1,
- event_message_id=next_message_id, inbound_message_id=next_inbound_id,
- channel_prompt=next_channel_prompt, message_type=next_message_type,
- persist_user_message=next_persist_message,
- persist_user_display_kind=next_display_kind,
- reply_expected=next_reply_expected,
- persist_user_display_metadata={
- **display_metadata_for_event(pending_event),
- **reply_expected_metadata(next_reply_expected), **diagnostic_metadata(pending_event)} or None,
- )
- except asyncio.CancelledError:
- await _run_followup_processing_hook(
- _hook_adapter, pending_event, "on_processing_complete", _followup_cancel_outcome(_hook_adapter))
- raise
- except BaseException:
- await _run_followup_processing_hook(
- _hook_adapter, pending_event, "on_processing_complete", ProcessingOutcome.FAILURE)
- raise
- await _run_followup_processing_hook(
- _hook_adapter, pending_event, "on_processing_complete", ProcessingOutcome.SUCCESS)
- merged = _preserve_queued_followup_history_offset(result, followup_result)
- # The TERMINAL turn of the chain owns the ledger identity for the outer final send, which
- # the adapter brackets against the event that OPENED the chain. Without this the terminal
- # reply is recorded under the first message's id, so a first reply that was refused (flood
- # control) has its outstanding row replaced and marked delivered by an identical-text
- # terminal reply, and is never redelivered. A deeper recursion has already set its own id,
- # so only fill the key while it is still absent: the innermost turn wins.
- if isinstance(merged, dict) and "queued_terminal_inbound_id" not in merged:
- merged = {
- **merged,
- "queued_terminal_inbound_id": next_inbound_id,
- "queued_terminal_display_kind": next_display_kind,
- "queued_terminal_reply_expected": next_reply_expected,
- "queued_terminal_source": next_source,
- "queued_terminal_notification_category": (
- (pending_event.metadata or {}).get("notification_category", "result")
- if pending_event is not None and pending_event.internal else "result"),
- }
- return merged
- async def _run_agent_cleanup_turn_tasks(
- self, turn_ctx: TurnContext, *, progress_task: Any, log_task: Any, interrupt_monitor: asyncio.Task,
- _notify_task: asyncio.Task, tracking_task: asyncio.Task, stream_task: Any,
- ) -> None:
- """``finally`` half of a turn: cancel background tasks, flush stream, release the session slot."""
- stream_consumer_holder, session_key = turn_ctx.stream_consumer_holder, turn_ctx.session_key
- for task in (progress_task, log_task, interrupt_monitor, _notify_task):
- if task:
- task.cancel()
- if stream_task:
- # No stream consumer was created: nothing to flush, cancel instead of waiting out 5s.
- if not (stream_consumer_holder and stream_consumer_holder[0] is not None):
- stream_task.cancel()
- with suppress(asyncio.CancelledError):
- await stream_task
- else:
- await self._await_stream_task(stream_task)
- # Abort + bounded wait for streaming TTS: covers paths where normal finalisation was skipped.
- _stts_finally = turn_ctx.streaming_tts_consumer_holder[0]
- # See #60671.
- if _stts_finally is not None and not _stts_finally.done:
- _stts_finally.abort("cleanup")
- with suppress(Exception):
- await _stts_finally.wait_complete(timeout=2.0)
- tracking_task.cancel()
- if session_key:
- # Release the slot only if this run's generation still owns it (/stop or /new may have
- # installed its own state).
- self._release_running_agent_state(session_key, run_generation=turn_ctx.run_generation)
- if self._draining:
- self._update_runtime_status("draining")
- for task in (progress_task, log_task, interrupt_monitor, tracking_task, _notify_task):
- if task:
- try:
- await task
- except asyncio.CancelledError:
- pass
- except Exception:
- # A background task that died of a real error must not abort the cleanup path.
- logger.debug("background turn task failed during cleanup", exc_info=True)
- async def _run_agent_edit_streamed_message(
- self, _sc, source, response, content, *, _sk, ok, fail_result: str, fail_exc: str,
- ) -> None:
- """Edit the stream consumer's message in place with ``content``; on success mark
- ``response["already_sent"]`` and log ``ok``. A returned failure logs ``fail_result`` as
- ``(session, error)`` and an exception logs ``fail_exc`` as ``(session, exc)``; either way
- ``already_sent`` stays unset so the normal final send delivers the content."""
- try:
- _res = await _sc.adapter.edit_message(
- chat_id=source.chat_id, message_id=_sc.message_id, content=content, finalize=True,
- )
- except Exception as _edit_err:
- logger.warning(fail_exc, _sk, _edit_err)
- return
- if not getattr(_res, "success", True):
- logger.warning(fail_result, _sk, getattr(_res, "error", None))
- return
- response["already_sent"] = True
- logger.info(*ok)
- async def _run_agent_mark_streamed_delivery(self, response: Any, turn_ctx: TurnContext) -> None:
- """Set ``response["already_sent"]`` when streaming already delivered the final reply.
- Never when the agent failed (unseen error), on "(empty)", or on a finite (`-z`, `chat -q`) turn:
- its viewer prints only the completion text, never the deltas (#139666). Both flags reflect call
- success, not content: a mismatch with the recorded turn-final payload never suppresses."""
- from gateway.session_finite import finite_turn_required
- _sc, source, session_key = turn_ctx.stream_consumer_holder[0], turn_ctx.source, turn_ctx.session_key
- if not isinstance(response, dict) or response.get("failed") or finite_turn_required() is True:
- return
- _final = response.get("final_response") or ""
- _is_empty_sentinel = not _final or _final == "(empty)"
- # response_previewed: only suppress if that EXACT text was delivered, not unrelated commentary.
- # Unrelated commentary/progress must not be mistaken for the final response (#14238).
- _previewed = bool(response.get("response_previewed"))
- _content_delivered = bool(_sc and getattr(_sc, "final_content_delivered", False))
- # #71643: a *successful* finalize edit can still carry only the last preview snapshot — deltas
- # generated between that edit and stream completion never reach any API call, and both suppression
- # flags are set from the call's success rather than its content. Reconcile the consumer's recorded
- # turn-final payload against the completed response: on a demonstrable mismatch (False) neither
- # final_response_sent nor final_content_delivered may suppress the normal final send. False also
- # covers payload-less multi-message split delivery (#78541). None (no record on a non-split legacy
- # path) keeps legacy trust; the failed-finalize family (#51828 / #33793) is unaffected because those
- # paths leave the flags False or record the complete fallback payload.
- _stale_finalized = False
- if _content_delivered and not _is_empty_sentinel:
- _matcher = getattr(_sc, "delivered_final_matches", None)
- if callable(_matcher):
- with suppress(Exception):
- _stale_finalized = _matcher(_final) is False
- if _stale_finalized:
- _content_delivered = False
- # Plugin hooks may append content after streaming finished — then send the final version.
- _transformed = bool(response.get("response_transformed"))
- # Suppress the normal send only when the actual final reply reached the user.
- _streamed = self._run_agent_stream_confirmed_final_delivery(_sc, _final, previewed=_previewed)
- if _is_empty_sentinel:
- return
- _sk = session_key or "?"
- if not _transformed and (_streamed or _content_delivered):
- logger.info(
- "Suppressing normal final send for session %s: final delivery already confirmed (streamed=%s previewed=%s content_delivered=%s).",
- _sk, _streamed, _previewed, _content_delivered,
- )
- response["already_sent"] = True
- elif not _transformed and _stale_finalized and _sc is not None:
- # Stale finalize: edit the streamed message up to the complete response (on failure the
- # normal send delivers). Not for split delivery — message_id is only the LAST chunk.
- _sc_msg_id = _sc.message_id
- if getattr(_sc, "_turn_split_delivery", False):
- logger.info(
- "Stale streamed finalize detected for session %s on a multi-message split; skipping the in-place reconciliation edit and delivering the complete response via normal final send (#78541).",
- _sk,
- )
- elif _sc_msg_id and _sc_msg_id != "__no_edit__" and getattr(_sc, "adapter", None) is not None:
- await self._run_agent_edit_streamed_message(
- _sc, source, response, _final, _sk=_sk,
- ok=("Reconciled stale streamed finalize for session %s: edited message %s with the complete response (#71643).", _sk, _sc_msg_id),
- fail_result="Stale-finalize reconciliation edit failed for session %s (%s); sending complete response via normal final send.",
- fail_exc="Stale-finalize reconciliation edit failed for session %s: %s; sending complete response via normal final send.",
- )
- else:
- logger.info(
- "Stale streamed finalize detected for session %s with no editable message; delivering complete response via normal final send (#71643).",
- _sk,
- )
- elif _transformed and _sc is not None:
- # Transformed after streaming: edit the streamed message instead of sending a duplicate.
- if _sc.message_id:
- await self._run_agent_edit_streamed_message(
- _sc, source, response, response["final_response"], _sk=_sk,
- ok=("Edited streamed message %s for session %s to include plugin-transformed content.", _sc.message_id, _sk),
- fail_result="Transformed-final edit failed for session %s (%s); sending transformed response via normal final send.",
- fail_exc="Failed to edit streamed message for session %s: %s",
- )
- elif _sc is not None and getattr(_sc, "stream_deltas_enabled", True):
- # DUPLICATE-RISK DIAGNOSTIC: a stream consumer existed but suppression did NOT fire; log
- # the decision inputs ("signal never set" vs "ack-pending race"). Skipped for consumers
- # never fed the final's deltas (interim-only wiring, #105341) — they cannot have raced
- # the normal final send, so the warning would be a guaranteed false positive.
- logger.warning(
- "Normal final-send NOT suppressed despite active stream consumer for session %s: "
- "streamed=%s previewed=%s content_delivered=%s transformed=%s final_len=%d — "
- "possible duplicate send (see wecom ack-timeout RCA).",
- _sk, _streamed, _previewed, _content_delivered, _transformed, len(_final),
- )
- def _run_agent_schedule_bubble_cleanup(self, response: Any, _cleanup_adapter: Any, turn_ctx: TurnContext) -> None:
- """Schedule deletion of tracked temporary progress bubbles after the final response lands.
- Failed runs keep them as breadcrumbs. Only on adapters with ``delete_message``; failures swallowed."""
- from gateway.run import safe_schedule_threadsafe
- _cleanup_msg_ids, session_key = turn_ctx._cleanup_msg_ids, turn_ctx.session_key
- if not (
- turn_ctx._cleanup_progress
- and _cleanup_adapter is not None
- and _cleanup_msg_ids
- and session_key
- and isinstance(response, dict)
- and not response.get("failed")
- and hasattr(_cleanup_adapter, "register_post_delivery_callback")
- ):
- return
- _ids_snapshot = list(_cleanup_msg_ids)
- _chat_id_snapshot = turn_ctx.source.chat_id
- _loop_snapshot = asyncio.get_running_loop()
- def _cleanup_temp_bubbles() -> None:
- async def _delete_all() -> None:
- for _mid in _ids_snapshot:
- with suppress(Exception):
- await _cleanup_adapter.delete_message(_chat_id_snapshot, _mid)
- with suppress(Exception):
- safe_schedule_threadsafe(
- _delete_all(), _loop_snapshot, logger=logger,
- log_message="Temp bubble cleanup scheduling error",
- )
- try:
- _cleanup_adapter.register_post_delivery_callback(
- session_key, _cleanup_temp_bubbles, generation=turn_ctx.run_generation,
- )
- except Exception as _rpe:
- logger.debug("Post-delivery cleanup registration failed: %s", _rpe)
- def _run_agent_bind_turn_wiring(
- self, turn_ctx: TurnContext, turn_runner: TurnRunner, source: SessionSource,
- event_message_id: Optional[str], _native_slack_task_cards: bool,
- ) -> Optional[dict[str, Any]]:
- """Resolve progress threading, then publish progress metadata and the sync→async bridges onto
- ``turn_ctx`` (the one-slot holders shared with run_sync's executor thread are TurnContext
- defaults). Returns ``_status_thread_metadata``."""
- turn_ctx._progress_metadata, turn_ctx._progress_reply_to, _status_thread_metadata = (
- self._run_agent_progress_threading(source, event_message_id, _native_slack_task_cards)
- )
- # Bridges: sync step/event/status callbacks → async hooks.emit and adapter.send.
- turn_ctx._loop_for_step = asyncio.get_running_loop()
- turn_ctx._hooks_ref = self.hooks
- turn_ctx._step_callback_sync = turn_runner._step_callback_sync
- turn_ctx._event_callback_sync = turn_runner._event_callback_sync
- turn_ctx._status_callback_sync = turn_runner._status_callback_sync
- turn_ctx._status_adapter = self._delivery_adapter_for(source)
- turn_ctx._status_chat_id = source.chat_id
- turn_ctx._status_thread_metadata = _status_thread_metadata
- return _status_thread_metadata
- async def _run_agent_notify_long_running(
- self, disp: GatewayRunner._RunAgentDisplay, turn_ctx: TurnContext, _executor_task_holder: list,
- ) -> None:
- """Periodic "still working" heartbeat, edited in place where supported. Stops once this run
- no longer owns the session slot or the executor finished. ``_executor_task_holder[0]`` is
- bound just after this task is scheduled (reads as None until then).
- Interval: agent.gateway_notify_interval / HERMES_AGENT_NOTIFY_INTERVAL (default 180s; 0 or
- long_running_notifications=off disables)."""
- from gateway.run import _float_env, _interim_metadata, _non_conversational_metadata
- _notify_start = time.time()
- _NOTIFY_INTERVAL = _float_env("HERMES_AGENT_NOTIFY_INTERVAL", 180)
- _long_running_mode = disp._display_surface_mode("long_running_notifications", default=True, allow_generic=True)
- if _NOTIFY_INTERVAL <= 0 or _long_running_mode == "off":
- return
- source, session_key, agent_holder = turn_ctx.source, turn_ctx.session_key, turn_ctx.agent_holder
- _status_thread_metadata = turn_ctx._status_thread_metadata
- _notify_adapter = self._delivery_adapter_for(source)
- if not _notify_adapter:
- return
- _heartbeat_msg_id: Optional[str] = None
- while True:
- await asyncio.sleep(_NOTIFY_INTERVAL)
- if not self._should_emit_long_running_notification(
- session_key, agent_holder[0], _executor_task_holder[0]
- ):
- break
- _elapsed_mins = int((time.time() - _notify_start) // 60)
- # Terse heartbeat by default; the iteration counter is gated on busy_ack_detail.
- _status_detail = ""
- _want_iteration_detail = bool(
- disp.resolve_display_setting(disp.user_config, disp.platform_key, "busy_ack_detail", True)
- )
- _a = self._agent_activity_summary(agent_holder[0])
- with suppress(Exception):
- if _a:
- _parts = []
- if _want_iteration_detail:
- _parts.append(format_iteration_progress(_a["api_call_count"], _a["max_iterations"]))
- _action = _a.get("current_tool") or _a.get("last_activity_desc")
- if _action:
- _parts.append(str(_action))
- if _parts:
- _status_detail = " — " + ", ".join(_parts)
- _heartbeat_text = (
- disp._generic_status_phrase("status")
- if _long_running_mode == "generic"
- else t("gateway.progress.working_heartbeat", minutes=_elapsed_mins, detail=_status_detail)
- )
- try:
- _notify_res = None
- if _heartbeat_msg_id:
- try:
- _notify_res = await _notify_adapter.edit_message(source.chat_id, _heartbeat_msg_id, _heartbeat_text)
- except Exception as _ee:
- logger.debug("Heartbeat edit failed: %s", _ee)
- _notify_res = None
- if not (_notify_res and getattr(_notify_res, "success", False)):
- # The edit above awaited; a drain/restart notice may have gone out meanwhile, and
- # a fresh "Working" bubble after it reads as a contradiction (#10990).
- if not self._should_emit_long_running_notification(
- session_key, agent_holder[0], _executor_task_holder[0]
- ):
- break
- _notify_res = await _notify_adapter.send(
- source.chat_id, _heartbeat_text,
- metadata=_interim_metadata(_non_conversational_metadata(_status_thread_metadata, platform=source.platform)),
- )
- if getattr(_notify_res, "success", False) and getattr(_notify_res, "message_id", None):
- _heartbeat_msg_id = str(_notify_res.message_id)
- if turn_ctx._cleanup_progress:
- turn_ctx._cleanup_msg_ids.append(_heartbeat_msg_id)
- except Exception as _ne:
- logger.debug("Long-running notification error: %s", _ne)
- async def _run_agent_inner(
- self, message: str, context_prompt: str, history: list[dict[str, Any]],
- source: SessionSource, session_id: str, session_key: str | None = None,
- run_generation: Optional[int] = None, _interrupt_depth: int = 0,
- event_message_id: Optional[str] = None, inbound_message_id: Optional[str] = None,
- channel_prompt: Optional[str] = None, moa_config: Optional[dict] = None,
- persist_user_message: Optional[Any] = None, persist_user_timestamp: Optional[float] = None,
- persist_user_display_kind: Optional[str] = None, message_type: Optional[str] = None,
- persist_user_display_metadata: Optional[dict] = None,
- reply_expected: Optional[bool] = None,
- scheduled_heartbeat: bool = False,
- title_user_message: Optional[str] = None,
- ) -> dict[str, Any]:
- """Run the agent; returns the full run_conversation result dict.
- Keys: "final_response", "messages", "api_calls", "completed"."""
- if self._get_proxy_url():
- result = await self._run_agent_via_proxy(
- message=message, context_prompt=context_prompt, history=history, source=source,
- session_id=session_id, session_key=session_key, run_generation=run_generation,
- event_message_id=event_message_id, scheduled_heartbeat=scheduled_heartbeat,
- )
- # The remote turn is this admission's executed turn (``TurnRunner.run_sync`` records
- # the local one): without it a finite run would settle as a no-execution refusal.
- from gateway.session_results import execution_result
- captured = execution_result.get()
- if captured is not None and 'result' not in captured:
- captured['result'] = result
- return result
- from run_agent import AIAgent
- disp = await asyncio.to_thread(self._run_agent_display_settings, source) # auth lock + config I/O; scope rides the copied context
- if scheduled_heartbeat:
- # A heartbeat is proactive work: tool chrome, drafts, thinking and periodic
- # liveness notices would create a user-visible ping before its final result is known.
- # Keep status callbacks intact for approvals and actionable failures. Interim messages keep the chat's
- # setting (TurnRunner drops them for a heartbeat): an agent this turn builds takes its mode from it.
- disp = dataclasses.replace(
- disp, tool_progress_enabled=False, _thinking_enabled=False, _native_slack_task_cards=False,
- needs_progress_queue=False,
- )
- turn_ctx, turn_runner, _cleanup_adapter = self._run_agent_build_turn_context(
- disp, AIAgent, message=message, source=source, session_key=session_key,
- run_generation=run_generation, context_prompt=context_prompt, history=history,
- session_id=session_id, _interrupt_depth=_interrupt_depth,
- event_message_id=event_message_id, inbound_message_id=inbound_message_id,
- channel_prompt=channel_prompt, moa_config=moa_config,
- title_user_message=title_user_message,
- persist_user_message=persist_user_message,
- persist_user_timestamp=persist_user_timestamp,
- persist_user_display_kind=persist_user_display_kind,
- reply_expected=reply_expected,
- persist_user_display_metadata=persist_user_display_metadata, scheduled_heartbeat=scheduled_heartbeat,
- voice_turn=str(getattr(message_type, "value", message_type) or "").lower() == "voice",
- )
- _status_thread_metadata = self._run_agent_bind_turn_wiring(
- turn_ctx, turn_runner, source, event_message_id, disp._native_slack_task_cards,
- )
- # Two independent quiet reasons: a muted diagnostic wake (ours) and a scheduled heartbeat.
- if not (scheduled_heartbeat or turn_ctx.mute_notification_reply):
- self._run_agent_start_streaming_tts(
- source, message_type, _status_thread_metadata, turn_ctx.streaming_tts_consumer_holder,
- )
- # Progress sender drains BOTH tool-progress lines and thinking bubbles (needs_progress_queue).
- spawn = asyncio.create_task
- progress_task = spawn(turn_runner.send_progress_messages()) if disp.needs_progress_queue else None
- log_task = spawn(self._run_agent_write_tool_log(disp.log_queue)) if disp.log_mode_enabled else None
- # The stream consumer is created inside run_sync; this task polls for it.
- stream_task = spawn(self._run_agent_stream_consumer_task(turn_ctx.stream_consumer_holder))
- tracking_task = spawn(self._run_agent_track_agent(turn_ctx))
- _interrupt_detected = asyncio.Event() # shared with backup check
- interrupt_monitor = spawn(self._run_agent_monitor_for_interrupt(turn_ctx, _interrupt_detected))
- # Periodic "still working" notifications so the user knows the agent hasn't died.
- _executor_task_holder: list = [None] # bound once the executor future exists (see below)
- _notify_task = (
- None if (scheduled_heartbeat or turn_ctx.mute_notification_reply or not automatic_notices_allowed(source))
- else spawn(self._run_agent_notify_long_running(disp, turn_ctx, _executor_task_holder))
- )
- try:
- # run_sync is TurnRunner.run_sync (bound method; executor call unchanged).
- worker = self._run_agent_start_turn_worker(turn_ctx, turn_runner.run_sync)
- _executor_task_holder[0] = worker.executor_task # read late by _notify_long_running
- response = await self._run_agent_await_turn_worker(worker, turn_ctx, _interrupt_detected, interrupt_monitor)
- if isinstance(response, dict):
- response["_notification_reply_muted"] = turn_ctx.mute_notification_reply
- self._run_agent_evict_on_fallback(turn_ctx)
- # Interrupted OR queued message (/queue)?
- result = turn_ctx.result_holder[0]
- adapter = self._delivery_adapter_for(source)
- await self._run_agent_finalize_streaming_tts(turn_ctx, adapter)
- pending_event, pending = await self._run_agent_drain_pending(result, adapter, source, session_key)
- if pending_event or pending:
- return await self._run_agent_queued_followup(
- turn_ctx, adapter, pending, pending_event, response, result, stream_task,
- )
- finally:
- await self._run_agent_cleanup_turn_tasks(
- turn_ctx, progress_task=progress_task, log_task=log_task, interrupt_monitor=interrupt_monitor,
- _notify_task=_notify_task, tracking_task=tracking_task, stream_task=stream_task,
- )
- await self._run_agent_mark_streamed_delivery(response, turn_ctx)
- self._run_agent_schedule_bubble_cleanup(response, _cleanup_adapter, turn_ctx)
- return response
|