run_turn_hygiene.py 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829
  1. """Gateway turn hygiene phases; inherited by GatewayTurnMixin."""
  2. from __future__ import annotations
  3. import logging
  4. from typing import TYPE_CHECKING
  5. import asyncio
  6. import dataclasses
  7. import inspect
  8. import json
  9. import os
  10. import queue
  11. import threading
  12. import time
  13. from agent.i18n import t
  14. from agent.session_activity import format_iteration_progress
  15. from contextlib import nullcontext, suppress
  16. from contextvars import copy_context
  17. from gateway.config import Platform
  18. from gateway.media_repair import repair_explicit_computer_use_media_paths
  19. from gateway.platforms.base import BasePlatformAdapter
  20. from gateway.platforms.event import MessageEvent
  21. from gateway.session import (
  22. SessionSource, _session_key_namespace, build_channel_continuity_note,
  23. build_session_context,
  24. )
  25. from gateway.session_transcript import TranscriptReadError
  26. from gateway.turn_context import TurnContext
  27. from gateway.turn_lease import DEFAULT_LEASE_WAIT, TurnLeaseTimeoutError
  28. from hermes_constants import get_hermes_home_override
  29. from pathlib import Path
  30. from typing import Any, Callable, Dict, List, Optional, Tuple
  31. from utils import base_url_hostname
  32. if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
  33. from gateway.run import GatewayRunner
  34. from gateway.run_turn_runner import TurnRunner
  35. # Log-record parity with the origin module.
  36. logger = logging.getLogger("gateway.run")
  37. class GatewayTurnHygieneMixin:
  38. @dataclasses.dataclass
  39. class _HygienePlan:
  40. """Hygiene pre-check outcome for one turn."""
  41. needs_compress: bool
  42. approx_tokens: int
  43. msg_count: int
  44. warn_token_threshold: int
  45. @staticmethod
  46. def _hmwa_hygiene_read_config(hs, data):
  47. """Apply model / compression knobs from the gateway config onto ``hs`` (invalid values keep the defaults)."""
  48. # Resolve model name (same logic as run_sync)
  49. _model_cfg = data.get("model", {})
  50. if isinstance(_model_cfg, str):
  51. hs.model = _model_cfg
  52. elif isinstance(_model_cfg, dict):
  53. hs.model = _model_cfg.get("default") or _model_cfg.get("model") or hs.model
  54. _raw_ctx = _model_cfg.get("context_length")
  55. if _raw_ctx is not None:
  56. with suppress(TypeError, ValueError):
  57. hs.config_context_length = int(_raw_ctx)
  58. hs.provider = _model_cfg.get("provider") or None
  59. hs.base_url = _model_cfg.get("base_url") or None
  60. # Only the enabled flag is shared with the agent's compression config (hygiene runs higher).
  61. _comp_cfg = data.get("compression", {})
  62. if not isinstance(_comp_cfg, dict):
  63. return
  64. hs.compression_enabled = str(_comp_cfg.get("enabled", True)).lower() in {"true", "1", "yes"}
  65. def _knob(key, current, cast, allow_zero=False):
  66. raw = _comp_cfg.get(key)
  67. if raw is None:
  68. return current
  69. try:
  70. parsed = cast(raw)
  71. except (TypeError, ValueError):
  72. return current
  73. return parsed if (parsed >= 0 if allow_zero else parsed > 0) else current
  74. hs.hard_msg_limit = _knob("hygiene_hard_message_limit", hs.hard_msg_limit, int)
  75. hs.timeout_seconds = _knob("hygiene_timeout_seconds", hs.timeout_seconds, float)
  76. hs.total_ceiling_seconds = _knob("hygiene_total_ceiling_seconds", hs.total_ceiling_seconds, float)
  77. # The ceiling can never be tighter than one idle window, or the extension loop would be dead code.
  78. hs.total_ceiling_seconds = max(hs.total_ceiling_seconds, hs.timeout_seconds)
  79. hs.max_turn_hold_seconds = _knob("hygiene_max_turn_hold_seconds", hs.max_turn_hold_seconds, float)
  80. hs.failure_cooldown_seconds = _knob(
  81. "hygiene_failure_cooldown_seconds", hs.failure_cooldown_seconds, float, allow_zero=True,
  82. )
  83. async def _hmwa_hygiene_settings(self, source, session_key):
  84. """Resolve model/provider/context-length + hygiene knobs (fail-soft: errors keep defaults).
  85. The 0.85 threshold is deliberately HIGHER than the agent's compressor (0.50): a safety net
  86. for sessions that grew between turns. ``max_turn_hold_seconds`` bounds the TURN wait
  87. (compressor keeps running detached, commit fenced); kept below transport idle-timeouts."""
  88. from gateway.run import _load_gateway_config
  89. hs = self._HygieneSettings(
  90. model="anthropic/claude-sonnet-4.6", threshold_pct=0.85, compression_enabled=True,
  91. hard_msg_limit=5000, timeout_seconds=30.0, total_ceiling_seconds=600.0,
  92. max_turn_hold_seconds=10.0, failure_cooldown_seconds=300.0, config_context_length=None,
  93. provider=None, base_url=None, api_key=None, data={},
  94. )
  95. try:
  96. hs.data = _load_gateway_config()
  97. if hs.data:
  98. self._hmwa_hygiene_read_config(hs, hs.data)
  99. configured_model, configured_provider, configured_base_url = hs.model, hs.provider, hs.base_url
  100. with suppress(Exception):
  101. hs.model, _hyg_runtime = self._resolve_session_agent_runtime(
  102. source=source, session_key=session_key,
  103. user_config=hs.data if isinstance(hs.data, dict) else None,
  104. )
  105. hs.provider = _hyg_runtime.get("provider") or hs.provider
  106. hs.base_url = _hyg_runtime.get("base_url") or hs.base_url
  107. hs.api_key = _hyg_runtime.get("api_key") or hs.api_key
  108. if hs.config_context_length is not None:
  109. try:
  110. from hermes_cli.route_identity import should_clear_context_pin_async
  111. if await should_clear_context_pin_async(
  112. configured_model, hs.model, configured_base_url, hs.base_url,
  113. configured_provider, hs.provider,
  114. ):
  115. hs.config_context_length = None
  116. except Exception:
  117. hs.config_context_length = None
  118. # custom_providers per-model context_length fallback (as in run_agent.py); needs base_url.
  119. if hs.config_context_length is None and hs.base_url:
  120. with suppress(TypeError, ValueError):
  121. try:
  122. from hermes_cli.config import (
  123. get_compatible_custom_providers as _gw_gcp,
  124. get_custom_provider_context_length as _gw_gccl,
  125. )
  126. _hyg_custom_providers = _gw_gcp(hs.data)
  127. except Exception:
  128. _hyg_custom_providers = hs.data.get("custom_providers")
  129. if not isinstance(_hyg_custom_providers, list):
  130. _hyg_custom_providers = []
  131. _hyg_custom_ctx = _gw_gccl(
  132. model=hs.model, base_url=hs.base_url, custom_providers=_hyg_custom_providers,
  133. )
  134. if _hyg_custom_ctx:
  135. hs.config_context_length = int(_hyg_custom_ctx)
  136. except Exception:
  137. pass
  138. return hs
  139. async def _hmwa_hygiene_plan(self, hs, history, session_entry, session_key):
  140. """Decide whether hygiene compression fires this turn (token/message thresholds, DB-backed
  141. failure cooldown, in-flight compression)."""
  142. from agent.model_metadata import estimate_messages_tokens_rough, get_model_context_length_async
  143. _hyg_context_length = await get_model_context_length_async(
  144. hs.model, base_url=hs.base_url or "", api_key=hs.api_key or "",
  145. config_context_length=hs.config_context_length, provider=hs.provider or "",
  146. )
  147. _compress_token_threshold = int(_hyg_context_length * hs.threshold_pct)
  148. _warn_token_threshold = int(_hyg_context_length * 0.95)
  149. _msg_count = len(history)
  150. # Real usage decides: the API-reported prompt count, else the anchor persisted on the session
  151. # row (real count + delta of what was appended since, survives gateway restarts), else the
  152. # rough estimate (runs 30-50% high, which only fires hygiene early — safe). Do NOT compensate
  153. # with a threshold multiplier.
  154. from agent.image_token_cost import image_cost_context, learned_image_token_cost
  155. _anchored = None
  156. # Images in any local delta/estimate are priced at the cost learned from this model's usage.
  157. with image_cost_context(learned_image_token_cost(hs.model, hs.base_url)):
  158. if session_entry.last_prompt_tokens <= 0:
  159. from agent.usage_anchor import persisted_anchor_tokens
  160. _session_db = getattr(self, "_session_db", None)
  161. _anchored = persisted_anchor_tokens(
  162. getattr(_session_db, "_db", _session_db), session_entry.session_id, history,
  163. )
  164. if session_entry.last_prompt_tokens > 0:
  165. _approx_tokens, _token_source = session_entry.last_prompt_tokens, "actual"
  166. elif _anchored is not None:
  167. _approx_tokens, _token_source = _anchored, "anchored"
  168. else:
  169. _approx_tokens, _token_source = estimate_messages_tokens_rough(history), "estimated"
  170. # Hard safety valve: force compression at an extreme message count regardless of tokens,
  171. # breaking the disconnect → no token data → no compression spiral. 5000 clears 1M+ sessions.
  172. _needs_compress = _approx_tokens >= _compress_token_threshold or _msg_count >= hs.hard_msg_limit
  173. if _needs_compress:
  174. # DB-backed cooldown (shared with context_compressor.py): survives gateway restarts, so a
  175. # failing compression is not re-triggered on every restart.
  176. # The in-memory dict was reset on every restart, re-triggering the same failing compression and
  177. # wedging session storage (#74136).
  178. _session_db = getattr(self, "_session_db", None)
  179. _getter = getattr(getattr(_session_db, "_db", _session_db), "get_compression_failure_cooldown", None)
  180. if _getter is not None:
  181. _cooldown_state = None
  182. with suppress(Exception):
  183. _cooldown_state = _getter(session_entry.session_id)
  184. if _cooldown_state and _cooldown_state.get("remaining_seconds", 0) > 0:
  185. logger.info(
  186. "Session hygiene: skipping compression for %s; "
  187. "previous failure cooldown active for %.1fs",
  188. session_entry.session_id, _cooldown_state["remaining_seconds"],
  189. )
  190. _needs_compress = False
  191. if _needs_compress and await self._session_has_compression_in_flight(session_key):
  192. # A prior compression still holds the durable lock (e.g. a shielded worker left by /stop):
  193. # another attempt would wait up to 600s behind a commit the fence will refuse.
  194. logger.info(
  195. "Session hygiene: skipping compression for %s; "
  196. "another compression is already in flight", session_entry.session_id,
  197. )
  198. _needs_compress = False
  199. if _needs_compress:
  200. logger.info(
  201. "Session hygiene: %s messages, ~%s tokens (%s) — auto-compressing "
  202. "(threshold: %s%% of %s = %s tokens)",
  203. _msg_count, f"{_approx_tokens:,}", _token_source,
  204. int(hs.threshold_pct * 100), f"{_hyg_context_length:,}", f"{_compress_token_threshold:,}",
  205. )
  206. return self._HygienePlan(_needs_compress, _approx_tokens, _msg_count, _warn_token_threshold)
  207. async def _hmwa_hygiene_wait_for_summary(self, attempt, hs, session_entry):
  208. """Progress-aware inline wait for the detached hygiene compressor. Returns the compressed
  209. transcript; raises ``HygieneTurnHoldExceeded`` (turn-hold budget) or
  210. ``asyncio.TimeoutError`` (idle/ceiling/fence cancel) for the caller's handlers. Idle timeout
  211. (fence ticks per streamed token) + hard ceiling (moved back by prompt reading) + turn-hold cap."""
  212. from agent.conversation_compression_deadline import effective_ceiling
  213. from gateway.run import HygieneTurnHoldExceeded, hygiene_wait_should_extend
  214. fence = attempt.commit_fence
  215. while True:
  216. if fence.is_cancelled:
  217. raise TimeoutError
  218. # Charge the idle budget from the LAST PROGRESS event, else silence can approach 2x timeout.
  219. _hyg_waited = time.monotonic() - attempt.wait_started
  220. _ceiling = effective_ceiling(fence, hs.total_ceiling_seconds, attempt.wait_started)
  221. _slice = min(max(hs.timeout_seconds - fence.seconds_since_progress(), 0.005),
  222. max(_ceiling - _hyg_waited, 0.005))
  223. # Cap the slice at the remaining turn-hold budget so a continuously-streaming worker can't
  224. # hold the turn until the ceiling. Budget exhausted → immediate timeout → abandonment.
  225. _turn_hold_remaining = hs.max_turn_hold_seconds - (time.monotonic() - attempt.wait_started)
  226. _slice = 0.005 if _turn_hold_remaining <= 0 else min(_slice, max(_turn_hold_remaining, 0.005))
  227. # Short poll so a /stop or /restart cancel is not stuck behind a full idle window.
  228. _idle_left = max(hs.timeout_seconds - fence.seconds_since_progress(), 0.005)
  229. _slice = min(_slice, 0.25)
  230. try:
  231. _compressed, _ = await asyncio.wait_for(asyncio.shield(attempt.future), timeout=_slice)
  232. return _compressed
  233. except TimeoutError:
  234. if fence.is_cancelled:
  235. raise
  236. _hyg_waited = time.monotonic() - attempt.wait_started
  237. _ceiling = effective_ceiling(fence, hs.total_ceiling_seconds, attempt.wait_started)
  238. _idle = fence.seconds_since_progress()
  239. # Never hold the TURN past the budget even while the summary streams: proceed on the
  240. # uncompressed transcript so the wire never trips a transport idle-timeout.
  241. if _hyg_waited >= hs.max_turn_hold_seconds:
  242. logger.info(
  243. "Session hygiene compression for session %s exceeded the turn-hold "
  244. "budget (%.1fs >= %.1fs) — abandoning inline wait, proceeding "
  245. "without compression this turn",
  246. session_entry.session_id, _hyg_waited, hs.max_turn_hold_seconds,
  247. )
  248. raise HygieneTurnHoldExceeded(
  249. f"turn-hold budget {hs.max_turn_hold_seconds:.1f}s "
  250. f"elapsed after {_hyg_waited:.1f}s"
  251. )
  252. if hygiene_wait_should_extend(
  253. idle=_idle, timeout=hs.timeout_seconds, waited=_hyg_waited,
  254. ceiling=_ceiling, fence_cancelled=fence.is_cancelled,
  255. ):
  256. if _slice >= _idle_left - 1e-9:
  257. logger.info(
  258. "Session hygiene compression for session %s still streaming after "
  259. "%.0fs (last progress %.1fs ago) — extending wait (ceiling %.0fs)",
  260. session_entry.session_id, _hyg_waited, _idle, _ceiling,
  261. )
  262. continue
  263. raise
  264. async def _hmwa_hygiene_cancel_or_adopt(self, attempt, context):
  265. """Cancel the worker at the commit fence; on success release its lease and defer agent
  266. cleanup, returning ``None``. When the worker already crossed into its commit, consume and
  267. return the compressed transcript instead (a successful compaction is never a timeout; the
  268. turn may be held past the budget by up to the commit duration — by design). The lock-free
  269. ``commit_in_flight`` marker keeps the poll from spinning on a hung commit."""
  270. fence = attempt.commit_fence
  271. while not fence.commit_in_flight:
  272. cancelled = fence.try_cancel_before_commit()
  273. if cancelled is None:
  274. await asyncio.sleep(0.025)
  275. elif cancelled:
  276. fence.release_cancelled_compression_lock()
  277. self._hmwa_hygiene_defer_cleanup(attempt, context)
  278. return None
  279. else:
  280. break
  281. _compressed, _ = await attempt.future
  282. return _compressed
  283. def _hmwa_hygiene_defer_cleanup(self, attempt, context):
  284. """Hand the agent's cleanup to the still-running worker future and mark it deferred."""
  285. self._defer_agent_cleanup_until_future_done(attempt.future, attempt.agent, context=context)
  286. attempt.cleanup_deferred = True
  287. @staticmethod
  288. def _hmwa_hygiene_stamp(agent, desc, provenance_name, debug_label):
  289. from agent.session_activity import ActivityProvenance
  290. from gateway.run import _stamp_hygiene_compression_provenance
  291. _stamp_hygiene_compression_provenance(agent, desc, getattr(ActivityProvenance, provenance_name), debug_label)
  292. async def _hmwa_hygiene_notify(self, source, meta, message, what):
  293. """Best-effort user notice on the hygiene thread; failure is logged, never raised."""
  294. from gateway.bot_peer_notices import automatic_notices_allowed
  295. try:
  296. _adapter = self._delivery_adapter_for(source) if automatic_notices_allowed(source) else None
  297. if _adapter and source.chat_id:
  298. await _adapter.emit_warning(source.chat_id, message, metadata=meta,
  299. logical_platform=source.platform)
  300. except Exception as _werr:
  301. logger.warning("Failed to deliver %s to user: %s", what, _werr)
  302. async def _hmwa_hygiene_record_failure_cooldown(self, hs, session_key, session_id, reason):
  303. """Escalate the failure streak (off-loop) and persist the cooldown, when enabled."""
  304. from gateway.run import _hygiene_cooldown_for_failure, _record_hygiene_cooldown
  305. if hs.failure_cooldown_seconds < 0:
  306. return
  307. _hyg_cooldown = await asyncio.to_thread(
  308. _hygiene_cooldown_for_failure, self, session_key, hs.failure_cooldown_seconds,
  309. )
  310. _record_hygiene_cooldown(self, session_id, _hyg_cooldown, reason)
  311. async def _hmwa_hygiene_on_turn_hold(self, attempt, hs, session_entry, session_key, source):
  312. """``except HygieneTurnHoldExceeded`` body: keep or cancel the worker's commit admission,
  313. notify the user, and re-raise; returns the compressed transcript only when the worker
  314. was already committing.
  315. Turn-hold expiry is an availability boundary, not a failure: the streak must NOT advance,
  316. only flat retry spacing is recorded. A watermark-fenced commit (rows appended after
  317. compression start survive as cloned tail) KEEPS admission: the turn proceeds uncompressed
  318. now and the summary is adopted at the worker's fenced commit — always cancelling burned
  319. every attempt for thinking summary models. Without the fence a late commit could clobber
  320. newer turns, so cancel."""
  321. from gateway.run import (
  322. _HYGIENE_TURNHOLD_RETRY_SECONDS, _record_hygiene_cooldown, _reset_hygiene_failure_streak
  323. )
  324. fence = attempt.commit_fence
  325. _hyg_keep_admission = bool(getattr(fence, "commit_watermark_fenced", False)) and not fence.is_cancelled
  326. if _hyg_keep_admission:
  327. self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene turn-hold")
  328. # NO retry-after here (it would also block the agent-side preflight compressor); spacing
  329. # comes from the durable compression lock. The done-callback records the flat retry-after
  330. # ONLY if the worker ends without committing anything.
  331. _sid, _skey, _agent = session_entry.session_id, session_key, attempt.agent
  332. def _hyg_adopt_or_space_retry(_fut, _gw=self, _sid=_sid, _skey=_skey, _agent=_agent):
  333. try:
  334. _exc = _fut.exception()
  335. except (asyncio.CancelledError, Exception):
  336. _committed = False
  337. else:
  338. _committed = _exc is None and (
  339. bool(getattr(_agent, "_last_compaction_in_place", False))
  340. or getattr(_agent, "session_id", _sid) != _sid
  341. )
  342. if _committed:
  343. logger.info(
  344. "Session hygiene compression for session %s finished after the "
  345. "turn-hold was released — summary adopted at the watermark-fenced "
  346. "commit boundary (#97963)", _sid,
  347. )
  348. try:
  349. _reset_hygiene_failure_streak(_gw, _skey)
  350. except Exception as _rs_err:
  351. logger.debug("hygiene streak reset after deferred adoption failed: %s", _rs_err)
  352. else:
  353. # Nothing to adopt (summary failed / fence refused / superseded): flat spacing so
  354. # sustained traffic doesn't spawn and abandon a compressor every turn.
  355. _record_hygiene_cooldown(
  356. _gw, _sid, _HYGIENE_TURNHOLD_RETRY_SECONDS,
  357. "hygiene compression deferred: turn-hold budget expired and the "
  358. "detached attempt did not commit",
  359. )
  360. attempt.future.add_done_callback(_hyg_adopt_or_space_retry)
  361. _log_suffix = (
  362. " — the watermark-fenced worker keeps its commit admission and the summary "
  363. "will be adopted when it finishes"
  364. )
  365. else:
  366. _adopted = await self._hmwa_hygiene_cancel_or_adopt(attempt, "session hygiene turn-hold")
  367. if _adopted is not None:
  368. return _adopted
  369. # Short flat retry-after, else every turn re-spawns, holds and cancels a compressor.
  370. _record_hygiene_cooldown(
  371. self, session_entry.session_id, _HYGIENE_TURNHOLD_RETRY_SECONDS,
  372. "hygiene compression deferred: turn-hold budget expired while the "
  373. "summary was still streaming",
  374. )
  375. _log_suffix = ""
  376. self._hmwa_hygiene_stamp(
  377. attempt.agent, "session hygiene compression turn-hold",
  378. "AGENT_COMPRESSION_TURNHOLD", "hygiene compression turn-hold activity stamp failed",
  379. )
  380. logger.info(
  381. "Session hygiene compression for session %s exceeded turn-hold budget (%.1fs); "
  382. "proceeding without compression this turn%s",
  383. session_entry.session_id, time.monotonic() - attempt.wait_started, _log_suffix,
  384. )
  385. await self._hmwa_hygiene_notify(
  386. source, attempt.meta, t("gateway.compress.turnhold_deferred"), "compression-turnhold notice",
  387. )
  388. raise
  389. async def _hmwa_hygiene_on_timeout(self, attempt, hs, session_entry, session_key, source):
  390. """``except asyncio.TimeoutError`` body: cancel at the commit fence, record the failure
  391. cooldown, warn the user, and re-raise; returns the compressed transcript only when the
  392. worker crossed the commit boundary first."""
  393. from gateway.run import _hygiene_compression_timeout_message
  394. fence = attempt.commit_fence
  395. _hyg_total_exhausted = fence.deadline_exceeded # armed with the ceiling before the wait began
  396. if _hyg_total_exhausted:
  397. # The worker checks this deadline between digest calls; keep its lease until it exits so
  398. # an unchanged session cannot overlap a retry (the release below is then a no-op).
  399. fence.retain_compression_lock_until_worker_done()
  400. # Capture fence state BEFORE try_cancel (which itself sets is_cancelled).
  401. _hyg_fence_cancelled = fence.is_cancelled
  402. _adopted = await self._hmwa_hygiene_cancel_or_adopt(attempt, "session hygiene timeout")
  403. if _adopted is not None:
  404. return _adopted
  405. await self._hmwa_hygiene_record_failure_cooldown(
  406. hs, session_key, session_entry.session_id,
  407. "session hygiene compression " + (
  408. "cancelled at commit fence" if _hyg_fence_cancelled
  409. else "total ceiling exhausted" if _hyg_total_exhausted
  410. else "timed out with no output from the summary model"
  411. ),
  412. )
  413. self._hmwa_hygiene_stamp(
  414. attempt.agent,
  415. "session hygiene compression cancelled at commit fence" if _hyg_fence_cancelled
  416. else "session hygiene compression timed out",
  417. "AGENT_COMPRESSION_TIMEOUT", "hygiene compression timeout activity stamp failed",
  418. )
  419. if _hyg_fence_cancelled:
  420. logger.warning(
  421. "Session hygiene compression for session %s was cancelled at the "
  422. "commit fence; continuing without compression", session_entry.session_id,
  423. )
  424. raise
  425. _hyg_elapsed = time.monotonic() - attempt.wait_started
  426. if _hyg_total_exhausted:
  427. logger.warning(
  428. "Session hygiene compression for session %s reached its total ceiling after "
  429. "%.1fs (progress observed=%s); continuing without compression",
  430. session_entry.session_id, _hyg_elapsed, fence.progress_observed,
  431. )
  432. else:
  433. logger.warning(
  434. "Session hygiene compression for session %s made no progress for %.1fs "
  435. "(total wait %.1fs, ceiling %.1fs); continuing without compression",
  436. session_entry.session_id, fence.seconds_since_progress(), _hyg_elapsed, hs.total_ceiling_seconds,
  437. )
  438. await self._hmwa_hygiene_notify(
  439. source, attempt.meta,
  440. _hygiene_compression_timeout_message(
  441. total_exhausted=_hyg_total_exhausted, elapsed=_hyg_elapsed,
  442. idle_timeout=hs.timeout_seconds, progress_observed=fence.progress_observed,
  443. ),
  444. "compression-timeout warning",
  445. )
  446. raise
  447. def _hmwa_hygiene_on_unwind(self, attempt, hs, session_entry, session_key):
  448. """``except BaseException`` body (caller re-raises): revoke commit admission BEFORE the host
  449. unwinds so the detached worker can never commit later, and record a cooldown — otherwise
  450. the next turn re-arms hygiene and waits up to 600s behind a fence that refuses again."""
  451. from gateway.run import _hygiene_cooldown_for_failure, _record_hygiene_cooldown
  452. attempt.commit_fence.revoke_commit_admission()
  453. if not attempt.cleanup_deferred:
  454. self._hmwa_hygiene_defer_cleanup(attempt, "session hygiene unwind")
  455. if hs.failure_cooldown_seconds >= 0:
  456. try:
  457. _record_hygiene_cooldown(
  458. self, session_entry.session_id,
  459. _hygiene_cooldown_for_failure(self, session_key, hs.failure_cooldown_seconds),
  460. "session hygiene compression cancelled at commit fence",
  461. )
  462. except Exception as _cd_err:
  463. logger.debug("hygiene unwind cooldown record failed: %s", _cd_err)
  464. async def _hmwa_hygiene_adopt_transcript(
  465. self, attempt, _compressed, history, plan, *, session_entry, source, _quick_key, run_generation,
  466. ):
  467. """Adopt a finished compression (rotation / in-place / refused); publishes the transcript to
  468. continue with on ``attempt.history``. Returns ``(rotated, in_place, new_count, new_tokens)``.
  469. Rewrite only on rotation (NEW session id): in-place compaction already soft-archived the
  470. old rows and rewrite_transcript() would DELETE them; neither rotation nor in-place signals
  471. FAILURE and an unconditional rewrite would leave only the summary. Write-before-repoint:
  472. a repoint-then-failed-rewrite would point the live entry at an empty session."""
  473. from gateway.run_turn import hygiene_no_commit_reason
  474. from agent.model_metadata import estimate_messages_tokens_rough
  475. _hyg_agent = attempt.agent
  476. # _compress_context rotates to a NEW session_id so the old transcript stays intact/searchable.
  477. _hyg_new_sid = _hyg_agent.session_id
  478. _hyg_rotated = _hyg_new_sid != session_entry.session_id
  479. _hyg_in_place = bool(getattr(_hyg_agent, "_last_compaction_in_place", False))
  480. # Anti-growth guard: refuse a compression that did not shrink the transcript (seen 427K→598K).
  481. _hyg_in_toks = estimate_messages_tokens_rough(history)
  482. _hyg_out_toks = estimate_messages_tokens_rough(_compressed)
  483. if _hyg_rotated and _hyg_out_toks > _hyg_in_toks:
  484. logger.warning(
  485. "Gateway hygiene compression for session %s would grow transcript (~%s -> ~%s "
  486. "tokens); keeping the original transcript unchanged",
  487. session_entry.session_id, f"{_hyg_in_toks:,}", f"{_hyg_out_toks:,}",
  488. )
  489. _hyg_rotated = False
  490. _compressed = history
  491. # Only persist a child transcript when rotation produced a NEW session id. In-place compaction does NOT
  492. # need a rewrite: archive_and_compact() has already soft-archived the previous active rows and
  493. # inserted the compacted messages as the new active set inside _compress_context(). Calling
  494. # rewrite_transcript() after in-place compaction would invoke replace_messages(active_only=False)
  495. # which DELETEs ALL rows — including the archived turns that archive_and_compact() deliberately
  496. # preserved (silent data loss, #61145). The danger this guards against (mirrors the /compress fix
  497. # #44794/#39704): if _compress_context returns a summary but neither rotates nor completes
  498. # archive_and_compact(), the session_id is unchanged for a FAILURE reason, and an unconditional
  499. # rewrite_transcript() would DELETE the original messages and replace them with only the compressed
  500. # summary (permanent data loss, #21301). Write-before-repoint (mirrors manual /compress): if we
  501. # repointed session_entry onto the child SID and rewrite_transcript then failed (lock/ENOSPC), the
  502. # live entry would already reference a brand-new empty session while the turn continues — the
  503. # conversation silently vanishes. Persist the child transcript first; only then rebind the live
  504. # entry.
  505. if _hyg_rotated:
  506. # Published child is already durable; a rewrite would drop rows cloned at publish.
  507. if not await self.async_session_store.persist_rotated_compression_child(
  508. session_entry.session_id, _hyg_new_sid, _compressed):
  509. logger.error(
  510. "Session hygiene: failed to persist compressed transcript for rotated session "
  511. "%s → %s; keeping the live entry on the original session so the "
  512. "conversation is not dropped", session_entry.session_id, _hyg_new_sid,
  513. )
  514. # Fail closed: treat like no rotation.
  515. _hyg_rotated = False
  516. _hyg_in_place = False
  517. else:
  518. session_entry.session_id = _hyg_new_sid
  519. # The held turn lease follows the rotation (alias keys still serialize on this turn).
  520. self._rebind_turn_lease(_quick_key, run_generation, _hyg_new_sid)
  521. await self.async_session_store._save()
  522. await asyncio.to_thread(
  523. self._sync_telegram_topic_binding, source, session_entry, reason="hygiene-compression",
  524. )
  525. if _hyg_rotated or _hyg_in_place:
  526. # Persisted (rotation) or persisted by archive_and_compact() (in-place): reset token count.
  527. session_entry.last_prompt_tokens = 0
  528. attempt.history = _compressed
  529. _new_count = len(_compressed)
  530. _new_tokens = estimate_messages_tokens_rough(_compressed)
  531. else:
  532. # No rewrite happened — post-compression counts equal the pre-compression ones.
  533. _new_count = plan.msg_count
  534. _new_tokens = plan.approx_tokens
  535. logger.warning(
  536. "Gateway hygiene compression for session %s did not rotate or compact in place (%s) — "
  537. "preserving the original transcript instead of overwriting it with the summary (#21301).",
  538. session_entry.session_id, hygiene_no_commit_reason(_hyg_agent),
  539. )
  540. logger.info(
  541. "Session hygiene: compressed %s → %s msgs, ~%s → ~%s tokens",
  542. plan.msg_count, _new_count, f"{plan.approx_tokens:,}", f"{_new_tokens:,}",
  543. )
  544. if _new_tokens >= plan.warn_token_threshold:
  545. logger.warning("Session hygiene: still ~%s tokens after compression", f"{_new_tokens:,}")
  546. return _hyg_rotated, _hyg_in_place, _new_count, _new_tokens
  547. async def _hmwa_hygiene_apply_result(
  548. self, attempt, hs, _compressed, history, plan, *,
  549. session_entry, session_key, source, _quick_key, run_generation,
  550. ):
  551. """Adopt a finished hygiene compression, rebind the session + turn lease, record
  552. streak/cooldown, and warn the user on abort."""
  553. from gateway.run import _reset_hygiene_failure_streak, hygiene_compaction_recovered
  554. _hyg_rotated, _hyg_in_place, _new_count, _new_tokens = await self._hmwa_hygiene_adopt_transcript(
  555. attempt, _compressed, history, plan, session_entry=session_entry, source=source,
  556. _quick_key=_quick_key, run_generation=run_generation,
  557. )
  558. # Summary failure aborts the compressor (nothing dropped). Warn the user visibly — agent.log
  559. # is invisible on TG/Discord — so they know the chat is "frozen" and can /compress or /reset.
  560. _comp = getattr(attempt.agent, "context_compressor", None)
  561. _hyg_aborted = _comp is not None and getattr(_comp, "_last_compress_aborted", False)
  562. # A fence-cancelled _compress_context returns the original transcript with
  563. # _last_compress_aborted False: treat that no-op as an abort so hygiene records a cooldown
  564. # instead of retrying into the 600s wait. A committed rotate/in-place is never an abort.
  565. _hyg_fence_cancelled = bool(attempt.commit_fence.is_cancelled and not _hyg_rotated and not _hyg_in_place)
  566. if _hyg_fence_cancelled:
  567. _hyg_aborted = True
  568. # Recovery decision lives in the unit-tested predicate: the "neither rotated nor in place"
  569. # path reuses pre-compression counts, so a numbers-only check would read a no-op as success.
  570. if not _hyg_aborted and hygiene_compaction_recovered(
  571. aborted=_hyg_aborted, rotated=_hyg_rotated, in_place=_hyg_in_place,
  572. msg_count=plan.msg_count, new_count=_new_count, approx_tokens=plan.approx_tokens,
  573. new_tokens=_new_tokens,
  574. ):
  575. await asyncio.to_thread(_reset_hygiene_failure_streak, self, session_key)
  576. if _hyg_aborted:
  577. await self._hmwa_hygiene_record_failure_cooldown(
  578. hs, session_key, session_entry.session_id,
  579. "session hygiene compression cancelled at commit fence" if _hyg_fence_cancelled
  580. else getattr(_comp, "_last_summary_error", None),
  581. )
  582. self._hmwa_hygiene_stamp(
  583. attempt.agent, "session hygiene compression aborted",
  584. "AGENT_COMPRESSION_COOLDOWN", "hygiene compression abort activity stamp failed",
  585. )
  586. if not _hyg_fence_cancelled:
  587. # Force-redact: provider exception text may contain credentials; this reaches users.
  588. from agent.redact import redact_sensitive_text
  589. _err = redact_sensitive_text(
  590. getattr(_comp, "_last_summary_error", None) or t("gateway.shared.unknown_error"), force=True)
  591. logger.warning("Session hygiene compression aborted: %s", _err)
  592. await self._hmwa_hygiene_notify(
  593. source, attempt.meta, t("gateway.compress.hygiene_failed"), "compression-failure warning",
  594. )
  595. # Configured aux model failed, recovered on the main model: only the user can fix that config.
  596. elif _comp is not None and getattr(_comp, "_last_aux_model_failure_model", None):
  597. _aux_model = getattr(_comp, "_last_aux_model_failure_model", "")
  598. _aux_err = getattr(_comp, "_last_aux_model_failure_error", None) or t("gateway.shared.unknown_error")
  599. await self._hmwa_hygiene_notify(
  600. source, attempt.meta, t("gateway.compress.aux_failed", model=_aux_model, error=_aux_err),
  601. "aux-model-fallback notice",
  602. )
  603. async def _hmwa_hygiene_codex_compaction(self, hs, plan, history, session_entry, session_key, _hyg_runtime):
  604. """codex app-server runtime: the real context is the server-side thread, not the transcript
  605. mirror. The detached-agent path would only rewrite the mirror and its finally-eviction
  606. would destroy the live thread (next turn starts blank), so use the cached agent's
  607. thread/compact/start and KEEP it cached."""
  608. from gateway.run import run_codex_hygiene_compaction
  609. # codex app-server runtime: the model's real context is the app-server's server-side thread, not the
  610. # transcript mirror. See #73503.
  611. _hyg_codex_auto = "native"
  612. _hyg_comp_cfg = hs.data.get("compression") if isinstance(hs.data, dict) else None
  613. if isinstance(_hyg_comp_cfg, dict):
  614. _hyg_codex_auto = str(_hyg_comp_cfg.get("codex_app_server_auto", "native") or "native")
  615. _hyg_codex_outcome = await run_codex_hygiene_compaction(
  616. self, session_key, session_entry.session_id, auto_mode=_hyg_codex_auto, history=history,
  617. approx_tokens=plan.approx_tokens, timeout_seconds=hs.total_ceiling_seconds,
  618. failure_cooldown_seconds=hs.failure_cooldown_seconds,
  619. )
  620. logger.info(
  621. "Session hygiene (codex app-server): %s (session=%s, mode=%s, ~%s tokens)",
  622. _hyg_codex_outcome, session_entry.session_id, _hyg_codex_auto, f"{plan.approx_tokens:,}",
  623. )
  624. async def _hmwa_hygiene_build_agent(self, _hyg_model, _hyg_runtime, session_entry):
  625. """Build the detached hygiene ``AIAgent`` with the live session's system prompt. Returns
  626. ``(agent, sync_session_db)``."""
  627. from gateway.run import _GATEWAY_HYGIENE_PLATFORM, _seed_hygiene_system_prompt
  628. from run_agent import AIAgent
  629. try:
  630. _hyg_session_row = await self._session_db.get_session(session_entry.session_id)
  631. except Exception as exc:
  632. _hyg_session_row = None
  633. logger.warning(
  634. "Session hygiene could not restore the system prompt for session %s: %s. "
  635. "Preserving an empty prompt so the live turn rebuilds it with its "
  636. "configured providers.", session_entry.session_id, exc, exc_info=True,
  637. )
  638. _hyg_session_db = getattr(self._session_db, "_db", self._session_db)
  639. # With compression.checkpoint_required on, load the memory provider so the checkpoint exists
  640. # before any mutation; otherwise keep the fast path (no provider init).
  641. from hermes_cli.config import load_config as _load_cfg
  642. from utils import is_truthy_value as _is_truthy
  643. _hyg_checkpoint_required = _is_truthy(
  644. ((_load_cfg() or {}).get("compression") or {}).get("checkpoint_required"), default=False,
  645. )
  646. _hyg_agent = AIAgent(
  647. **_hyg_runtime, model=_hyg_model, max_iterations=4, quiet_mode=True,
  648. skip_memory=not _hyg_checkpoint_required, enabled_toolsets=["memory"],
  649. session_id=session_entry.session_id, session_db=_hyg_session_db,
  650. )
  651. _seed_hygiene_system_prompt(_hyg_agent, _hyg_session_row)
  652. # The stamp only marks this agent as no real surface. Since #104414 Platform is not a
  653. # restore-identity field, so it no longer forces the next live turn to rebuild; the seed's
  654. # retain flag is what keeps the reduced-toolset build out of the session row (#122822).
  655. _hyg_agent.platform = _GATEWAY_HYGIENE_PLATFORM
  656. return _hyg_agent, _hyg_session_db
  657. async def _hmwa_hygiene_detached_attempt(
  658. self, attempt, hs, plan, history, _hyg_msgs, _hyg_model, _hyg_runtime,
  659. source, session_entry, session_key, _quick_key, run_generation,
  660. ):
  661. """Run one detached hygiene compression attempt end to end; publishes the transcript to
  662. continue with (compressed or original) on ``attempt.history``."""
  663. from gateway.run import HygieneTurnHoldExceeded
  664. from agent.conversation_compression import CompressionCommitFence
  665. _hyg_agent, _hyg_session_db = await self._hmwa_hygiene_build_agent(_hyg_model, _hyg_runtime, session_entry)
  666. attempt.agent = _hyg_agent
  667. try:
  668. # Hygiene owns the session binding, so prefer in-place compaction over minting a
  669. # continuation child. Without a SessionDB this stays False.
  670. _hyg_agent.compression_in_place = True
  671. _bind_hyg_state = getattr(getattr(_hyg_agent, "context_compressor", None), "bind_session_state", None)
  672. if callable(_bind_hyg_state):
  673. _bind_hyg_state(_hyg_session_db, session_entry.session_id)
  674. # Never finalize on close() — that would end the live gateway session row.
  675. _hyg_agent._end_session_on_close = False
  676. _hyg_agent._print_fn = lambda *a, **kw: None
  677. loop = asyncio.get_running_loop()
  678. _hyg_commit_fence = CompressionCommitFence(total_ceiling_seconds=hs.total_ceiling_seconds)
  679. # Default executor (NOT self._get_executor): a hung summary must never occupy an
  680. # agent-work slot. MUST run in the caller's contextvars (multiplex secret scope).
  681. attempt.commit_fence = _hyg_commit_fence
  682. attempt.future = loop.run_in_executor(
  683. None,
  684. # But it MUST run inside the caller's contextvars: under multiplex_profiles the profile
  685. # secret scope / HERMES_HOME override live in ContextVars, and a bare run_in_executor worker
  686. # starts with an empty Context — the summary model's get_secret(<PROVIDER>_API_KEY) then
  687. # fails closed (UnscopedSecretError) and every hygiene compaction silently degrades to a
  688. # lossy truncation (#100849 bundle).
  689. copy_context().run,
  690. # task_id = the live turn's dedup bucket (session row id), never "" (= reset every task).
  691. lambda: _hyg_agent._compress_context(
  692. _hyg_msgs, "", approx_tokens=plan.approx_tokens, commit_fence=_hyg_commit_fence,
  693. task_id=session_entry.session_id or "default", trigger="gateway_hygiene",
  694. ),
  695. )
  696. # Register the live worker with shutdown NOW, not only once it is deferred: the default
  697. # executor is outside self._executor's quiesce, so an untracked in-flight summary would let
  698. # stop() close/checkpoint state.db under its late write (mirrors run_codex_hygiene_compaction).
  699. self._track_deferred_agent_worker(attempt.future, _hyg_agent)
  700. attempt.wait_started = time.monotonic()
  701. try:
  702. _compressed = await self._hmwa_hygiene_wait_for_summary(attempt, hs, session_entry)
  703. except HygieneTurnHoldExceeded:
  704. _compressed = await self._hmwa_hygiene_on_turn_hold(attempt, hs, session_entry, session_key, source)
  705. except TimeoutError:
  706. _compressed = await self._hmwa_hygiene_on_timeout(attempt, hs, session_entry, session_key, source)
  707. except BaseException:
  708. self._hmwa_hygiene_on_unwind(attempt, hs, session_entry, session_key)
  709. raise
  710. await self._hmwa_hygiene_apply_result(
  711. attempt, hs, _compressed, history, plan, session_entry=session_entry,
  712. session_key=session_key, source=source, _quick_key=_quick_key,
  713. run_generation=run_generation,
  714. )
  715. finally:
  716. # Evict the cached agent so the next turn rebuilds its system prompt.
  717. self._evict_cached_agent(session_key)
  718. if not attempt.cleanup_deferred:
  719. await self._cleanup_agent_resources_off_loop(_hyg_agent, context="session hygiene")
  720. async def _hmwa_run_session_hygiene(
  721. self, event, source, session_entry, session_key, history, _quick_key, run_generation,
  722. ):
  723. """Auto-compress pathologically large transcripts before the agent starts so oversized
  724. histories don't cause repeated truncation/context failures. Token source: the API's
  725. prompt_tokens from the last turn, else a char/4 estimate."""
  726. from gateway.run import HygieneTurnHoldExceeded
  727. if not history or len(history) < 4:
  728. return history
  729. hs = await self._hmwa_hygiene_settings(source, session_key)
  730. # Hygiene can never land with compression disabled; a sub-limit transcript is the identity (#111988).
  731. if not hs.compression_enabled:
  732. return self._bound_hygiene_payload(history, hs, session_entry)
  733. plan = await self._hmwa_hygiene_plan(hs, history, session_entry, session_key)
  734. # No compression this turn (under both thresholds, cooldown, or one already in flight): without
  735. # the bound the model would get the full uncompressed transcript.
  736. if not plan.needs_compress:
  737. return self._bound_hygiene_payload(history, hs, session_entry)
  738. attempt = self._HygieneAttempt(agent=None, meta=self._event_thread_metadata(event, source), history=history)
  739. try:
  740. _hyg_model, _hyg_runtime = self._resolve_session_agent_runtime(
  741. source=source, session_key=session_key,
  742. user_config=hs.data if isinstance(hs.data, dict) else None,
  743. )
  744. if str(_hyg_runtime.get("api_mode") or "").lower() == "codex_app_server":
  745. await self._hmwa_hygiene_codex_compaction(hs, plan, history, session_entry, session_key, _hyg_runtime)
  746. elif _hyg_runtime.get("api_key"):
  747. # Pass the FULL transcript (tool results included) as the agent loop does: filtering
  748. # to user/assistant starved the compressor (tool results are the bulk of context).
  749. _hyg_msgs = [m for m in history if m.get("role") in {"user", "assistant", "tool"}]
  750. if len(_hyg_msgs) >= 4:
  751. await self._hmwa_hygiene_detached_attempt(
  752. attempt, hs, plan, history, _hyg_msgs, _hyg_model, _hyg_runtime,
  753. source, session_entry, session_key, _quick_key, run_generation,
  754. )
  755. except HygieneTurnHoldExceeded:
  756. # Availability boundary, not a failure — already logged at INFO by the turn-hold handler.
  757. # Must not hit the generic "auto-compress failed" warning below: that log is how thinking-model
  758. # deployments read as permanently broken (#97963; surfaced by @686f6c61 in PR #99657).
  759. pass
  760. except Exception as e:
  761. logger.warning("Session hygiene auto-compress failed: %s", e)
  762. # A landed compression published a NEW transcript on attempt.history: leave it byte-identical.
  763. # Anything else (turn-hold, timeout, unwind, codex path) left the FULL uncompressed transcript
  764. # there — that is the fail-closed case (#111988).
  765. if attempt.history is history:
  766. return self._bound_hygiene_payload(history, hs, session_entry)
  767. return attempt.history
  768. @staticmethod
  769. def _bound_hygiene_payload(history, hs, session_entry):
  770. """``bound_model_input_without_hygiene`` over ``hs.hard_msg_limit``, with one INFO line when the
  771. cut is real. Below the limit this is the identity — no allocation, no behaviour change."""
  772. from gateway.run_turn import bound_model_input_without_hygiene
  773. bounded = bound_model_input_without_hygiene(history, hs.hard_msg_limit)
  774. if bounded is not history:
  775. logger.info(
  776. "Session hygiene did not land for %s: bounding the model payload to %s of %s "
  777. "messages (hard limit %s) — the stored transcript is unchanged",
  778. session_entry.session_id, len(bounded), len(history), hs.hard_msg_limit,
  779. )
  780. return bounded