run_turn_persistence.py 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752
  1. """Gateway turn persistence phases; inherited by GatewayTurnMixin."""
  2. from __future__ import annotations
  3. import logging
  4. from typing import TYPE_CHECKING
  5. import asyncio
  6. import dataclasses
  7. import inspect
  8. import json
  9. import os
  10. import queue
  11. import threading
  12. import time
  13. from agent.i18n import t
  14. from agent.turn_failure_copy import (
  15. FAILED_TURN_DISPLAY_KIND, FAILED_TURN_NOTICE, OPEN_TURN_TAIL_ROLES, PARTIAL_FAILED_TURN_NOTICE,
  16. )
  17. from agent.session_activity import format_iteration_progress
  18. from contextlib import nullcontext, suppress
  19. from contextvars import copy_context
  20. from gateway.bot_peer_notices import automatic_notices_allowed
  21. from gateway.config import Platform
  22. from gateway.media_repair import repair_explicit_computer_use_media_paths
  23. from gateway.platforms.base import BasePlatformAdapter
  24. from gateway.platforms.event import MessageEvent
  25. from gateway.session import (
  26. SessionSource, _session_key_namespace, build_channel_continuity_note,
  27. build_session_context,
  28. )
  29. from gateway.session_detached import detached_route_of
  30. from gateway.session_transcript import TranscriptReadError
  31. from gateway.turn_context import TurnContext
  32. from gateway.warning_notifications import diagnostic_wake_muted
  33. from gateway.turn_lease import DEFAULT_LEASE_WAIT, TurnLeaseTimeoutError
  34. from hermes_constants import get_hermes_home_override
  35. from pathlib import Path
  36. from typing import Any, Callable, Dict, List, Optional, Tuple
  37. from utils import base_url_hostname
  38. if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
  39. from gateway.run import GatewayRunner
  40. from gateway.run_turn_runner import TurnRunner
  41. # Log-record parity with the origin module.
  42. logger = logging.getLogger("gateway.run")
  43. def _acp_launch(runner, source) -> bool:
  44. """An editor session: ACP shows no reasoning event yet, so its reply keeps the projected block."""
  45. from gateway.session_setting_commands import local_policy
  46. from hermes_state_runtime import RuntimeStoreError
  47. try:
  48. return getattr(local_policy(runner, source), "source", None) == "acp"
  49. except RuntimeStoreError: # a local route with no restored policy is not an editor launch
  50. return False
  51. class GatewayTurnPersistenceMixin:
  52. @staticmethod
  53. def _is_intentional_silence(agent_result, response) -> bool:
  54. try:
  55. from gateway.response_filters import is_intentional_silence_agent_result
  56. return is_intentional_silence_agent_result(agent_result, response)
  57. except Exception:
  58. return False
  59. async def _record_room_silence_decision(
  60. self, session_key, agent_result, silent, *, display_kind=None, reply_expected=None,
  61. ):
  62. """Bind recovery permission to this exact persisted final and active-turn token.
  63. A raw persisted marker is not proof that a partial/interrupted turn completed. A crash
  64. before this receipt is durable keeps the human safeguard; transcript bytes stay intact.
  65. """
  66. from gateway.response_filters import is_intentional_silence_response, silence_allowed
  67. kind = agent_result.get("queued_terminal_display_kind", display_kind)
  68. expected = agent_result.get("queued_terminal_reply_expected", reply_expected)
  69. if (not silent or not session_key or silence_allowed(kind, expected)
  70. or not is_intentional_silence_response(agent_result.get("final_response"))):
  71. return
  72. store = self.async_session_store
  73. try:
  74. entry = await store.lookup_by_session_key(session_key)
  75. token = entry.active_turn_token if entry is not None else None
  76. if not isinstance(token, str):
  77. return
  78. history = await store.load_transcript(entry.session_id)
  79. # The agent may have saved its final before gateway session metadata was appended.
  80. # Match recovery's visible tail rather than the literal last bookkeeping row.
  81. last = next((m for m in reversed(history) if m.get("role") not in ("session_meta", "system")), {})
  82. if (last.get("role") != "assistant" or last.get("timestamp") is None
  83. or last.get("content") != agent_result.get("final_response")):
  84. return
  85. await store.set_session_metadata(session_key, "room_silence_decision", {
  86. "token": token, "timestamp": last.get("timestamp"), "silent": bool(silent),
  87. })
  88. except Exception:
  89. # The receipt only lets crash recovery honour this silence; without it recovery keeps
  90. # the human-turn notice. Live delivery must stay silent rather than become an error.
  91. logger.warning("Could not record the room silence decision for %s", session_key, exc_info=True)
  92. async def _hmwa_shape_agent_response(
  93. self, agent_result, source, history, session_entry, session_key,
  94. _quick_key, run_generation, _run_start_session_id, _platform_name, _msg_start_time,
  95. persist_user_display_kind: Optional[str] = None,
  96. reply_expected: Optional[bool] = None,
  97. ):
  98. """Turn the raw agent result into the outbound text: sentinel/silence handling, response
  99. logging, resume-pending clear, empty-response normalization, and identity-guarded
  100. post-compression session_id propagation. Returns
  101. ``(response, _intentional_silence, agent_messages)``."""
  102. from gateway.run import (
  103. _is_gateway_hidden_reasoning_incomplete_turn, _normalize_empty_agent_response,
  104. _sanitize_gateway_final_response, _should_clear_resume_pending_after_turn,
  105. )
  106. response = agent_result.get("final_response") or ""
  107. # Hidden-reasoning-only retry exhaustion: the loop's sentinel text doubles as final_response
  108. # and would be delivered verbatim (peer agents would ingest it as a completed turn).
  109. if _is_gateway_hidden_reasoning_incomplete_turn(agent_result):
  110. response = ""
  111. _intentional_silence = self._is_intentional_silence(agent_result, response)
  112. # A queued (/queue) chain's TERMINAL turn owns the silence verdict, not the event that
  113. # opened the chain: an internal follow-up, or a message not addressed to the bot, may go
  114. # silent; any other human one must not.
  115. from gateway.response_filters import is_machinery_display_kind, strip_trailing_silence_marker
  116. from gateway.run_turn import _unexpected_silence_reply
  117. _silence_kind = agent_result.get("queued_terminal_display_kind", persist_user_display_kind)
  118. _silence_reply_expected = agent_result.get("queued_terminal_reply_expected", reply_expected)
  119. _silence_source = agent_result.get("queued_terminal_source", source)
  120. # A bot peer expects no reply even when it addressed us (a DM, a mention): its message may end
  121. # the exchange in silence, and the visible fallback would be runtime text it answers (#137027).
  122. if not automatic_notices_allowed(_silence_source):
  123. _silence_reply_expected = False
  124. if _intentional_silence and not self._silence_allowed_for_source(
  125. _silence_source, _silence_kind, _silence_reply_expected, agent_result,
  126. ):
  127. logger.warning(
  128. "silence marker rejected on a user turn: platform=%s chat=%s",
  129. _platform_name, source.chat_id or "unknown",
  130. )
  131. _intentional_silence = False
  132. response = _unexpected_silence_reply()
  133. elif _intentional_silence and not is_machinery_display_kind(_silence_kind):
  134. logger.debug(
  135. "silence marker suppressed on an unaddressed or opted-in turn: platform=%s chat=%s",
  136. _platform_name, source.chat_id or "unknown",
  137. )
  138. # "(empty)" = the model produced no visible content after exhausting all retries. One
  139. # text with the CLI explainer and the desktop (agent/turn_explainers.py) so the user
  140. # reads the same words on every surface.
  141. if not automatic_notices_allowed(source) and (response == "(empty)" or agent_result.get("failed")):
  142. # A failed or empty turn's text is runtime failure copy, never the model's answer: a bot
  143. # peer would take it as a message to answer (#137027).
  144. response = ""
  145. if response == "(empty)" and not _intentional_silence:
  146. from agent.turn_explainers import EMPTY_RESPONSE_EXPLANATION
  147. _model = str(agent_result.get("model") or "").strip() or t("gateway.errors.empty_response_model_label")
  148. response = t("gateway.shared.warn_passthrough", error=EMPTY_RESPONSE_EXPLANATION.format(model=_model))
  149. agent_messages = agent_result.get("messages", [])
  150. logger.info(
  151. "response ready: platform=%s chat=%s session=%s time=%.1fs api_calls=%d response=%d chars",
  152. _platform_name, source.chat_id or "unknown", session_key or "unknown",
  153. time.time() - _msg_start_time, agent_result.get("api_calls", 0), len(response),
  154. )
  155. # Successful turn: clear the consecutive-restart stuck-loop counter and resume_pending (set
  156. # by drain-timeout shutdown) so later messages don't get the restart-interruption note.
  157. if session_key and _should_clear_resume_pending_after_turn(agent_result):
  158. await self._clear_restart_failure_count(session_key)
  159. try:
  160. await self.async_session_store.clear_resume_pending(session_key)
  161. except Exception as _e:
  162. logger.debug("clear_resume_pending failed for %s: %s", session_key, _e, exc_info=True)
  163. # Normalize empty responses: surface errors, partial failures, and work-without-text.
  164. # Fix for #18765.
  165. if not _intentional_silence:
  166. response = strip_trailing_silence_marker(response)
  167. # A bot peer gets the model's own words only: the runtime's retry hints and "not
  168. # processed" lines are new inbound messages for its turn loop (#137027).
  169. if automatic_notices_allowed(source):
  170. response = _normalize_empty_agent_response(agent_result, response, history_len=len(history))
  171. response = _sanitize_gateway_final_response(source.platform, response)
  172. # The agent thread already updated the contextvar; propagate to SessionEntry + _save() only
  173. # if the binding still points at the session this run was launched against.
  174. if agent_result.get("session_id") and agent_result["session_id"] != session_entry.session_id:
  175. if session_entry.session_id == _run_start_session_id:
  176. session_entry.session_id = agent_result["session_id"]
  177. # The held turn lease follows the rotation (persistence writes to the NEW id).
  178. self._rebind_turn_lease(_quick_key, run_generation, session_entry.session_id)
  179. await self.async_session_store._save()
  180. await self.async_session_store._record_gateway_session_peer(
  181. session_entry.session_id, session_key, source,
  182. )
  183. await asyncio.to_thread(
  184. self._sync_telegram_topic_binding, source, session_entry, reason="agent-result-compression",
  185. )
  186. else:
  187. logger.info(
  188. "Skipping agent-result session split sync for %s because the session binding "
  189. "moved from %s to %s before compression finished",
  190. session_key or "?", _run_start_session_id, session_entry.session_id,
  191. )
  192. return response, _intentional_silence, agent_messages
  193. # reasoning_style → (header catalog key, per-line quote prefix for blank / non-blank lines)
  194. _REASONING_QUOTE_STYLES = {
  195. "subtext": ("gateway.reasoning.quote_label_discord", "-# ", "-#"),
  196. "blockquote": ("gateway.reasoning.quote_label_md", "> ", ">"),
  197. }
  198. def _hmwa_prepend_reasoning(self, agent_result, response, source, _intentional_silence):
  199. """Prepend the last reasoning block when show_reasoning is on for this platform. Mattermost
  200. requires an explicit per-platform opt-in (scratch text, not final-answer content)."""
  201. if source.platform == Platform.LOCAL and not _acp_launch(self, source):
  202. # Local viewers receive reasoning.delta through their event stream; projecting it into
  203. # the final response duplicates it and leaks messaging-platform formatting into Desktop.
  204. return response
  205. from gateway.run import _load_gateway_config, _platform_config_key, _resolve_gateway_display_bool
  206. try:
  207. _show_reasoning_effective = _resolve_gateway_display_bool(
  208. _load_gateway_config(), _platform_config_key(source.platform), "show_reasoning",
  209. default=bool(getattr(self, "_show_reasoning", False)), platform=source.platform,
  210. require_platform_override_for={Platform.MATTERMOST},
  211. )
  212. except Exception:
  213. _show_reasoning_effective = (
  214. False if source.platform == Platform.MATTERMOST else getattr(self, "_show_reasoning", False)
  215. )
  216. last_reasoning = agent_result.get("last_reasoning")
  217. if not (_show_reasoning_effective and response and not _intentional_silence and last_reasoning):
  218. return response
  219. from gateway.stream_consumer_fences import escape_code_fences_for_display
  220. # Collapse long reasoning to keep messages readable
  221. lines = last_reasoning.strip().splitlines()
  222. if len(lines) > 15:
  223. display_reasoning = "\n".join(lines[:15]) + t("gateway.reasoning.more_lines", count=len(lines) - 15)
  224. else:
  225. display_reasoning = last_reasoning.strip()
  226. # Per-platform render style: Discord defaults to "-# " subtext, others keep the code block.
  227. try:
  228. from gateway.display_config import resolve_display_setting
  229. _reasoning_style = resolve_display_setting(
  230. _load_gateway_config(), _platform_config_key(source.platform), "reasoning_style", "code",
  231. )
  232. except Exception:
  233. _reasoning_style = "code"
  234. _quote = self._REASONING_QUOTE_STYLES.get(_reasoning_style)
  235. if _quote:
  236. header_key, prefix, empty = _quote
  237. _quoted = "\n".join(f"{prefix}{ln}" if ln else empty for ln in display_reasoning.splitlines())
  238. return f"{t(header_key)}\n{_quoted}\n\n{response}"
  239. # Escape ``` inside reasoning so inner fences don't break the outer code block.
  240. display_reasoning = escape_code_fences_for_display(display_reasoning)
  241. return t("gateway.reasoning.block", reasoning=display_reasoning, response=response)
  242. def _hmwa_runtime_footer_line(self, agent_result, source, _turn_seconds):
  243. """Runtime-metadata footer for the FINAL message of the turn; off by default
  244. (display.runtime_footer.enabled=false). Never on a finite (`-z`, `chat -q`) turn: its reply is
  245. the printed answer itself."""
  246. from gateway.run import _load_gateway_config, _platform_config_key, _terminal_scope_cwd
  247. from gateway.session_finite import finite_turn_required
  248. # A bot peer gets the model's words only: a streamed reply sends the footer as its own message.
  249. if finite_turn_required() is True or not automatic_notices_allowed(source):
  250. return ""
  251. try:
  252. from gateway.runtime_footer import build_footer_line as _bfl
  253. return _bfl(
  254. user_config=_load_gateway_config(),
  255. platform_key=_platform_config_key(source.platform), model=agent_result.get("model"),
  256. context_tokens=agent_result.get("last_prompt_tokens", 0) or 0,
  257. context_length=agent_result.get("context_length") or None,
  258. cwd=_terminal_scope_cwd(""), turn_seconds=_turn_seconds,
  259. requested_model=agent_result.get("requested_model"),
  260. served_model=agent_result.get("served_model"),
  261. )
  262. except Exception as _footer_err:
  263. logger.debug("runtime_footer build failed: %s", _footer_err)
  264. return ""
  265. async def _hmwa_post_turn_hooks(self, hook_ctx, agent_result, response):
  266. """agent:end hook, process-watcher scheduling, and watch-notification drain."""
  267. await self.hooks.emit("agent:end", {
  268. **hook_ctx, "response": (response or "")[:500], "model": agent_result.get("model", ""),
  269. "provider": agent_result.get("provider", ""),
  270. })
  271. # Pending process watchers (check_interval on background processes)
  272. try:
  273. from tools.process_registry import process_registry
  274. # Detach the batch atomically (reassign, not clear()) so concurrent appends aren't dropped.
  275. watchers = process_registry.pending_watchers
  276. process_registry.pending_watchers = []
  277. for i, watcher in enumerate(watchers):
  278. asyncio.create_task(self._run_process_watcher(watcher))
  279. if i % 100 == 99:
  280. await asyncio.sleep(0)
  281. except Exception as e:
  282. logger.error("Process watcher setup error: %s", e)
  283. # Drain watch notifications that arrived during the run; the queue also carries process /
  284. # async-delegation completions owned elsewhere — inject only watch-type events.
  285. try:
  286. from tools.process_registry import process_registry as _pr
  287. await self._drain_watch_notifications(_pr.completion_queue)
  288. except Exception as e:
  289. logger.debug("Watch queue drain error: %s", e)
  290. # One owner for the boundary copy (agent/turn_failure_copy.py): the core closer in
  291. # agent/conversation_loop.py writes the same row on the paths that never reach this layer.
  292. def _hmwa_add_failed_turn_notice(self, response, notice):
  293. """Make failed-turn delivery explicit without replacing the provider-specific guidance."""
  294. response = str(response or "").strip()
  295. return f"{response}\n\n{notice}" if response else notice
  296. def _hmwa_failed_turn_notice(self, agent_result):
  297. """Choose retry guidance without assuming completed tool effects can be repeated safely."""
  298. from gateway.media_repair import _current_turn_messages
  299. # Compression during the failed turn can move the slice boundary; the shared helper falls
  300. # back to the last user row so tool evidence is not silently dropped.
  301. turn_messages = _current_turn_messages(
  302. agent_result.get("messages", []) or [], agent_result.get("history_offset", 0),
  303. )
  304. if any(
  305. message.get("role") == "tool"
  306. or (message.get("role") == "assistant" and message.get("tool_calls"))
  307. for message in turn_messages
  308. ):
  309. return PARTIAL_FAILED_TURN_NOTICE
  310. return FAILED_TURN_NOTICE
  311. async def _hmwa_close_failed_turn(self, session_id, notice):
  312. """Append the gateway-owned assistant boundary iff the durable tail is open (a user row, or the
  313. tool row of a call that ran before the failure).
  314. The tail, not "did the gateway write the user row", is the key: on the primary path the
  315. agent's turn-start flush already persisted the row (so the platform-id dedupe skips the
  316. gateway write), and a platform redelivery of an already-closed turn must not stack a
  317. second assistant row."""
  318. if await self.async_session_store.transcript_tail_role(session_id) not in OPEN_TURN_TAIL_ROLES:
  319. return
  320. await self.async_session_store.append_to_transcript(session_id, {
  321. "role": "assistant", "content": notice, "timestamp": time.time(), "display_kind": FAILED_TURN_DISPLAY_KIND,
  322. })
  323. def _hmwa_classify_turn_failure(self, agent_result, history, session_entry):
  324. """Classify a finished turn for transcript persistence. Returns
  325. ``(agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure)``.
  326. Context-overflow failures must NOT persist the user message (session would grow and
  327. reproduce the failure forever); transient failures (429/timeout/5xx) DO."""
  328. from gateway.run import _is_gateway_hidden_reasoning_incomplete_turn
  329. # Save the full conversation to the transcript, including tool calls. This preserves the complete
  330. # agent loop (tool_calls, tool results, intermediate reasoning) so sessions can be resumed with full
  331. # context and transcripts are useful for debugging and training data. IMPORTANT: For
  332. # context-overflow failures (compression exhausted, generic 400 on large sessions) we must NOT
  333. # persist the user's message — doing so would grow the session further and cause the same failure on
  334. # the next attempt, an infinite loop. (#1630, #9893) Transient failures (429, timeout, connection
  335. # error, provider 5xx) are different: the session is not oversized, and silently dropping the user
  336. # message causes severe context loss on retry — the agent forgets what was just asked. Persist the
  337. # user turn so the conversation is preserved. (#7100)
  338. agent_failed_early = bool(agent_result.get("failed"))
  339. hidden_reasoning_incomplete = _is_gateway_hidden_reasoning_incomplete_turn(agent_result)
  340. from gateway.run_turn import is_context_overflow_failure_result
  341. is_context_overflow_failure = is_context_overflow_failure_result(agent_result, len(history))
  342. if is_context_overflow_failure:
  343. logger.info(
  344. "Skipping transcript persistence for context-overflow "
  345. "failure in session %s to prevent session growth loop.", session_entry.session_id,
  346. )
  347. elif agent_failed_early:
  348. logger.info(
  349. "Transient agent failure in session %s — persisting user "
  350. "message so conversation context is preserved on retry.", session_entry.session_id,
  351. )
  352. elif hidden_reasoning_incomplete:
  353. logger.warning(
  354. "Suppressing hidden-reasoning-only incomplete gateway turn for session %s: %s",
  355. session_entry.session_id, agent_result.get("error", "processing incomplete"),
  356. )
  357. return agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure
  358. async def _hmwa_compression_exhaustion_reset(
  359. self, agent_result, response, session_entry, session_key, source,
  360. ):
  361. """Auto-reset a permanently oversized session so the next message starts fresh instead of
  362. replaying the oversized context forever. Never on a lock-contended defer — that is the
  363. OPPOSITE case (a concurrent path holds the lock and is shrinking it). Returns
  364. ``(response, session_entry)``."""
  365. # When compression is exhausted, the session is permanently too large to process. (#9893) Never wipe
  366. # the session for that — retry-next-message semantics apply (#69870 lock-skip consumer; salvaged
  367. # from #49874).
  368. if agent_result.get("compression_deferred"):
  369. logger.info(
  370. "Compression deferred for session %s — the compression "
  371. "lock is held by a concurrent compressor. Keeping the "
  372. "session intact; the next message retries normally.",
  373. session_entry.session_id if session_entry else "?",
  374. )
  375. elif agent_result.get("compression_exhausted") and (
  376. getattr(source, "platform", None) == Platform.API_SERVER or detached_route_of(source)):
  377. # The caller addresses an API conversation by its id and every API door re-validates that
  378. # binding (``restore_api_session``): a rotated entry would refuse each later request to it
  379. # (``admission_conflict``). A conversation its chat moved past has no route of its own to
  380. # reset (gateway/session_detached.py): its app viewer gets the start-fresh guidance a
  381. # failed overflow turn gives. Never a reset that did not happen.
  382. logger.info("Keeping session %s after compression exhaustion.",
  383. session_entry.session_id if session_entry else "?")
  384. from gateway.run import _context_overflow_reply
  385. guidance = _context_overflow_reply() if detached_route_of(source) else ""
  386. if guidance not in (response or ""):
  387. response = f"{response}\n\n{guidance}" if response else guidance
  388. elif agent_result.get("compression_exhausted") and session_entry and session_key:
  389. if not self._hmwa_defer_exhaustion_reset(session_entry, session_key, source):
  390. session_entry = await self._hmwa_rotate_exhausted_session(session_entry, session_key, source)
  391. response = (response or "") + t("gateway.session.auto_reset_context_exhausted")
  392. return response, session_entry
  393. def _hmwa_defer_exhaustion_reset(self, session_entry, session_key, source):
  394. """True when the reset was handed to the authority drain. A LOCAL session (the Bot Chat a
  395. ``hermes peer dm`` lands in) resets its target in the store, which refuses while an admission
  396. on it is started (``session_busy``): this turn's own. The drain runs the reset right after it
  397. settles this admission, before it claims any successor (the Bot Chat's retry included)."""
  398. from gateway.session_ingress import executing_admission
  399. if getattr(source, "platform", None) != Platform.LOCAL or not executing_admission.get():
  400. return False
  401. from gateway.session_authorities import active_authority
  402. authority = active_authority(self)
  403. if authority is None:
  404. return False
  405. authority.deferred_resets[session_key] = lambda: self._hmwa_rotate_exhausted_session(
  406. session_entry, session_key, source)
  407. return True
  408. async def _hmwa_rotate_exhausted_session(self, session_entry, session_key, source):
  409. """The compression-exhaustion auto-reset itself; returns the route's (fresh) entry."""
  410. logger.info("Auto-resetting session %s after compression exhaustion.", session_entry.session_id)
  411. from gateway.session_automation import move_route_wakes, reset_and_rehome_messages
  412. # Messages queued behind the exhausted turn still run, on the fresh session (unlike /new).
  413. new_entry, release_fresh = await reset_and_rehome_messages(self, session_key, session_entry.session_id)
  414. try:
  415. await move_route_wakes(self, session_key, session_entry.session_id)
  416. self._evict_cached_agent(session_key)
  417. # The /queue overflow behind the deferred head slot is queued user input, like the
  418. # re-homed native rows: it survives the boundary the funnel below otherwise clears.
  419. queued = list(self._overflow_queue(session_key) or ())
  420. # Conversation boundary: the funnel clears every conversation-scoped per-session dict.
  421. self._clear_conversation_scope(session_key, reason="compression_exhausted_reset")
  422. if queued:
  423. self._session_state(session_key).conversation.queued_events.extend(queued)
  424. if new_entry is not None:
  425. # Re-point the Telegram topic binding at the fresh session, or the binding-heal walk
  426. # switches the next message back onto the bloated child and re-triggers exhaustion
  427. # forever. No-op on non-topic lanes.
  428. # Compression rotated session_entry.session_id to the oversized compressed child earlier
  429. # this turn (the agent-result sync above), and that _sync also rewrote the (chat_id,
  430. # thread_id) -> bloated-child binding. reset_session swaps in a clean, parentless session,
  431. # but without re-syncing the binding the next inbound message in this topic gets
  432. # switch_session'd back onto the bloated child by the binding-heal walk, reloads the
  433. # oversized transcript, and re-triggers compression exhaustion forever (#35809 — regression
  434. # of the #9893/#10063 auto-reset).
  435. session_entry = new_entry
  436. await asyncio.to_thread(
  437. self._sync_telegram_topic_binding, source, session_entry, reason="compression-exhausted-reset",
  438. )
  439. finally:
  440. # Only now may the fresh session's FIFO claim (it is fenced until here, against every
  441. # scheduler): after eviction, scope clear and binding sync.
  442. release_fresh()
  443. return session_entry
  444. @staticmethod
  445. def _hmwa_user_transcript_entry(event, prepared, ts):
  446. """Transcript row for the inbound user turn (clean text + event time when captured)."""
  447. # Transient failure (429/timeout/5xx): persist only the user message so the next message can load a
  448. # transcript that reflects what was said. Skip the assistant error text since it's a
  449. # gateway-generated hint, not model output. Hidden- reasoning-only incomplete turns follow the same
  450. # persistence rule so peer-agent channels don't ingest them as completed assistant turns. (#7100,
  451. # #51628)
  452. _user_entry = {
  453. "role": "user",
  454. "content": (
  455. prepared.persist_user_message if prepared.persist_user_message is not None
  456. else prepared.message_text
  457. ),
  458. "timestamp": prepared.persist_user_timestamp if prepared.persist_user_timestamp is not None else ts,
  459. }
  460. if prepared.persist_user_display_kind:
  461. _user_entry["display_kind"] = prepared.persist_user_display_kind
  462. from gateway.response_filters import persisted_user_metadata_for_event
  463. display_metadata = {**({"gateway_input_owner": prepared.persistence_owner} if prepared.persistence_owner else {}),
  464. **persisted_user_metadata_for_event(event)}
  465. if display_metadata:
  466. _user_entry["display_metadata"] = display_metadata
  467. if getattr(event, "message_id", None):
  468. _user_entry["message_id"] = str(event.message_id)
  469. return _user_entry
  470. async def _hmwa_persist_turn_transcript(
  471. self, *, event, source, session_entry, session_key, agent_result, agent_messages,
  472. prepared, response, agent_failed_early, hidden_reasoning_incomplete, is_context_overflow_failure,
  473. ):
  474. """Persist this turn to the transcript (session_meta on first turn, user-only on transient
  475. failure, nothing on context overflow), update last_prompt_tokens, and re-baseline the
  476. cached agent's message count."""
  477. from gateway.run import _resolve_gateway_model
  478. ts = time.time() # Unix epoch float — consistent with DB storage
  479. store = self.async_session_store
  480. sid = session_entry.session_id
  481. history = prepared.history
  482. # The agent already persisted this turn's rows (codex app-server reports agent_persisted=True
  483. # too); skip the DB write. Default = a session DB exists; non-persisting runtimes pass False.
  484. # The agent already persisted these messages to SQLite via _flush_messages_to_session_db(), so skip
  485. # the DB write here to prevent the duplicate-write bug (#860 / #42039). This holds for the codex
  486. # app-server runtime too: although it early-returns and bypasses conversation_loop's per-step
  487. # flushes, it flushes its own projected assistant/tool messages before returning and reports
  488. # agent_persisted=True (see agent/codex_runtime.py). Reading the flag (default = self._session_db is
  489. # not None) keeps the persistence contract explicit and lets any future non-persisting runtime opt
  490. # into a gateway-side write by returning False.
  491. agent_persisted = agent_result.get("agent_persisted", self._session_db is not None)
  492. _user_row = self._hmwa_user_transcript_entry(event, prepared, ts)
  493. if is_context_overflow_failure:
  494. pass # Skip all transcript writes — don't grow a broken session
  495. else:
  496. if not history and source.platform != Platform.LOCAL:
  497. # Fresh session: the tool definitions (as sent in the API request) make the transcript
  498. # self-describing. A LOCAL route (one-shot, chat -q, ACP, TUI attach) keeps the classic
  499. # in-process CLI's transcript shape: the marker is gateway bookkeeping every local
  500. # reader filters out, and the session row already carries model/model_config.
  501. await store.append_to_transcript(sid, {
  502. "role": "session_meta",
  503. "tools": agent_result.get("tools", []) or [],
  504. "model": _resolve_gateway_model(),
  505. "platform": source.platform.value if source.platform else "",
  506. "timestamp": ts,
  507. })
  508. if agent_failed_early or hidden_reasoning_incomplete:
  509. # Transient failure / hidden-reasoning incomplete: persist only the user message (the
  510. # assistant error text is a gateway hint, not model output). Dedupe on platform
  511. # message_id (Telegram retries after transient failures).
  512. if event.message_id and await store.has_platform_message_id(sid, str(event.message_id)):
  513. logger.info(
  514. "Skipping duplicate user turn (message_id=%s) in session %s",
  515. event.message_id, sid,
  516. )
  517. else:
  518. await store.append_to_transcript(sid, _user_row, skip_db=agent_persisted)
  519. # Close the failed turn: a user-only tail lets alternation repair merge this request
  520. # into an unrelated future message and replay stale side effects (#107070).
  521. await self._hmwa_close_failed_turn(sid, self._hmwa_failed_turn_notice(agent_result))
  522. else:
  523. # Only the NEW messages: history_offset (what the agent saw), not len(history), which
  524. # counts session_meta entries stripped before the agent saw them.
  525. history_len = agent_result.get("history_offset", len(history))
  526. new_messages = agent_messages[history_len:] if len(agent_messages) > history_len else []
  527. if not new_messages:
  528. # Edge case: fall back to simple user/assistant rows.
  529. await store.append_to_transcript(sid, _user_row, skip_db=agent_persisted)
  530. if response:
  531. await store.append_to_transcript(
  532. sid, {"role": "assistant", "content": response, "timestamp": ts},
  533. skip_db=agent_persisted,
  534. )
  535. else:
  536. # Attach the inbound platform message_id to the first user entry so platform-level
  537. # quote-resolution (e.g. Yuanbao) can find earlier @bot messages by original id.
  538. _user_msg_id_attached = False
  539. for msg in new_messages:
  540. if msg.get("role") == "system":
  541. continue # rebuilt each run
  542. entry = {**msg, "timestamp": ts}
  543. if (
  544. not _user_msg_id_attached
  545. and msg.get("role") == "user"
  546. and event.message_id
  547. and "message_id" not in entry
  548. ):
  549. entry["message_id"] = str(event.message_id)
  550. _user_msg_id_attached = True
  551. await store.append_to_transcript(sid, entry, skip_db=agent_persisted)
  552. # The agent persists token counts/model itself; keep only last_prompt_tokens for hygiene.
  553. await store.update_session(
  554. session_entry.session_key, last_prompt_tokens=agent_result.get("last_prompt_tokens", 0),
  555. touch_activity=not bool(getattr(event, "internal", False)),
  556. )
  557. # Re-baseline the cached agent's message_count now that ALL of this turn's writes are done:
  558. # the coherence guard snapshots at agent-BUILD time, so our own writes would otherwise
  559. # trigger a rebuild next turn (destroying prompt caching).
  560. await self._refresh_agent_cache_message_count(session_key, sid)
  561. @staticmethod
  562. def _usage_window_chat_wording(agent_result: dict, response: str) -> str:
  563. """A usage-window refusal sent to a chat: credits steps point at the Portal (a chat's /topup has no
  564. Keep going control) and the reset reads as a duration (the gateway's clock is not the reader's).
  565. Every other reply is returned unchanged."""
  566. if not (agent_result.get("usage_window_message") and response):
  567. return response
  568. from agent.plan_limits import chat_refusal_wording
  569. return chat_refusal_wording(response, agent_result.get("failure_resets_at"))
  570. async def _deliver_turn_response_then_limit_offer(
  571. self, event, source, session_entry, session_key, run_generation,
  572. agent_result, agent_messages, response, footer_line, intentional_silence,
  573. ):
  574. """``_hmwa_deliver_turn_response``, then, for a usage-window refusal, the limit offer: the limit
  575. sentence is the reply, and the button follows it once it lands."""
  576. delivered = await self._hmwa_deliver_turn_response(
  577. event, source, session_entry, session_key, run_generation,
  578. agent_result, agent_messages, response, footer_line, intentional_silence,
  579. )
  580. if (delivered or agent_result.get("already_sent")) and not intentional_silence:
  581. self._note_bot_reply(source)
  582. if isinstance(agent_result.get("usage_limit"), dict):
  583. self._schedule_usage_limit_offer(event, agent_result["usage_limit"], session_key=session_key,
  584. generation=run_generation)
  585. return delivered
  586. async def _hmwa_deliver_turn_response(
  587. self, event, source, session_entry, session_key, run_generation,
  588. agent_result, agent_messages, response, _footer_line, _intentional_silence,
  589. ):
  590. """Final delivery decisions: intentional silence, voice reply, streamed-turn media/footer.
  591. Returns the text for the adapter to send, or ``None`` when already delivered."""
  592. if diagnostic_wake_muted(event):
  593. return None
  594. # Intentional silence is a delivery decision: the [SILENT] turn stays persisted (alternation).
  595. if _intentional_silence:
  596. logger.info("Suppressing intentional silence marker for session %s", session_entry.session_id)
  597. response = ""
  598. adapter = self._delivery_adapter_for(source)
  599. # Auto voice reply (TTS audio before the text) unless streaming TTS already delivered audio.
  600. _streaming_tts_done = adapter is not None and bool(
  601. getattr(adapter, "_streaming_tts_turn_completed", lambda *_a, **_k: False)(session_key, run_generation)
  602. )
  603. if not _streaming_tts_done and self._should_send_voice_reply(
  604. event, response, agent_messages, already_sent=bool(agent_result.get("already_sent")),
  605. ):
  606. await self._send_voice_reply(event, response)
  607. # Streamed responses still need MEDIA: files delivered (chunks carry the tags verbatim). Never
  608. # skip when the agent failed: the error text is new content streaming didn't show.
  609. if agent_result.get("already_sent") and not agent_result.get("failed"):
  610. # The queued-follow-up lane uploads this response's attachments itself; re-scanning here
  611. # would upload every file a second time.
  612. if response and adapter and not agent_result.get("media_already_delivered"):
  613. await self._deliver_media_from_response(response, event, adapter)
  614. # Streaming delivered the body, but the footer was held back (`not already_sent` gate).
  615. if _footer_line and adapter and not _intentional_silence:
  616. try:
  617. await adapter.send(source.chat_id, _footer_line, metadata=self._event_thread_metadata(event, source))
  618. except Exception as _e:
  619. logger.debug("trailing footer send failed: %s", _e)
  620. # Return None so the body isn't sent twice; stash the delivered text on the event for the
  621. # /loop and /goal hooks that read the return value.
  622. with suppress(Exception):
  623. event._streamed_final_response = str(response or "")
  624. return None
  625. return response
  626. # Chat-side next steps keyed by HTTP status; Hermes commands only (/login is the gateway's own
  627. # sign-in, `{relogin}` the profile-aware host equivalent, filled from the turn's agent provider).
  628. # Values are catalog keys (``gateway.errors.hint_*``); 401 carries a ``{relogin}`` placeholder.
  629. _STATUS_HINTS = {
  630. 401: "gateway.errors.hint_auth",
  631. 402: "gateway.errors.hint_quota",
  632. 529: "gateway.errors.hint_overloaded",
  633. }
  634. async def _hmwa_agent_error_reply(self, e, event, source, session_entry, session_key, prepared):
  635. """``except Exception`` body of the agent turn: stop typing, log, persist the inbound user
  636. turn once, and build the sanitized user-facing error reply."""
  637. # Retain Slack thread/workspace routing so a failed turn cannot leave its status visible.
  638. await self._hmwa_stop_typing_for_turn(event, source)
  639. logger.exception("Agent error in session %s", session_key)
  640. status_code = getattr(e, "status_code", None)
  641. if status_code in {400, 500} and len(prepared.history) > 50:
  642. # Context overflow / payload too large: a deterministic rejection (#107567), and the same
  643. # no-grow rule as the persist path (#1630) — nothing is written into an oversized session.
  644. from gateway.run import _context_overflow_reply
  645. from gateway.session_results import record_unexecuted_failure
  646. return record_unexecuted_failure(_context_overflow_reply())
  647. # Replay can coalesce inputs; only this input's durable marker establishes ownership.
  648. try:
  649. if prepared.message_text is not None and session_entry is not None:
  650. _owned = await self.async_session_store.has_input_owner(
  651. prepared.persistence_session_id, prepared.persistence_owner,
  652. )
  653. if not _owned:
  654. await self.async_session_store.append_to_transcript(
  655. session_entry.session_id, self._hmwa_user_transcript_entry(event, prepared, time.time()),
  656. )
  657. # Tool effects are unknown after an exception.
  658. await self._hmwa_close_failed_turn(session_entry.session_id, PARTIAL_FAILED_TURN_NOTICE)
  659. except Exception:
  660. logger.debug("Failed to persist inbound user message after agent exception", exc_info=True)
  661. # Never expose raw exception types/messages to end users (info-leakage risk).
  662. _hint_key = self._STATUS_HINTS.get(status_code)
  663. status_hint = t(_hint_key) if _hint_key and status_code != 401 else ""
  664. if status_code == 401:
  665. from agent.turn_failure_copy import relogin_command_hint
  666. _turn_agent = getattr(self._session_state(session_key).turn, "agent", None)
  667. status_hint = t(_hint_key, relogin=relogin_command_hint(getattr(_turn_agent, "provider", None)))
  668. elif status_code == 429:
  669. # Plan usage limit (resets on a schedule) vs a transient rate limit
  670. _err_json = {}
  671. with suppress(Exception):
  672. _err_json = e.response.json().get("error", {})
  673. if not isinstance(_err_json, dict):
  674. _err_json = {}
  675. _resets_in = _err_json.get("resets_in_seconds")
  676. if _err_json.get("type") != "usage_limit_reached":
  677. status_hint = t("gateway.errors.hint_rate_limited")
  678. elif _resets_in and _resets_in > 0:
  679. import math
  680. status_hint = t("gateway.errors.hint_usage_limit_resets", hours=math.ceil(_resets_in / 3600))
  681. else:
  682. status_hint = t("gateway.errors.hint_usage_limit")
  683. elif status_code == 400:
  684. status_hint = t("gateway.errors.hint_rejected")
  685. from gateway.session_results import record_unexecuted_failure
  686. if not automatic_notices_allowed(source):
  687. # The failure is recorded for the operator; the bot peer gets no apology to answer (#137027).
  688. record_unexecuted_failure(t("gateway.errors.generic_failed"))
  689. return None
  690. if getattr(e, "reason", None) == "launch_credentials_unavailable":
  691. # The session's own route key is gone or changed: /retry repeats it, /new is the way out.
  692. from agent.turn_failure_copy import launch_credentials_copy
  693. return record_unexecuted_failure(launch_credentials_copy(e), 'launch_credentials_unavailable')
  694. # A local client (CLI, TUI, Desktop) is the operator at this host: its result carries the
  695. # real cause, so its details line is not the chat apology with the log pointer cut off.
  696. local_error = f"{type(e).__name__}: {e}"[:500] if source.platform == Platform.LOCAL else None
  697. return record_unexecuted_failure(self._hmwa_add_failed_turn_notice(
  698. t("gateway.errors.generic_failed_with_hint", hint=status_hint), PARTIAL_FAILED_TURN_NOTICE,
  699. ), error=local_error)
  700. def _hmwa_discard_stale_result(self, source, _quick_key, run_generation):
  701. """A newer run generation superseded this turn: drop its deferred post-delivery callback."""
  702. logger.info(
  703. "Discarding stale agent result for %s — generation %d is no longer current",
  704. _quick_key or "?", run_generation,
  705. )
  706. self._pop_post_delivery_callback(self._delivery_adapter_for(source), _quick_key, run_generation)