run_turn_prepare.py 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753
  1. """Gateway turn prepare phases; inherited by GatewayTurnMixin."""
  2. from __future__ import annotations
  3. import logging
  4. from typing import TYPE_CHECKING
  5. import asyncio
  6. import dataclasses
  7. import inspect
  8. import json
  9. import os
  10. import queue
  11. import threading
  12. import time
  13. from agent.i18n import t
  14. from agent.session_activity import format_iteration_progress
  15. from contextlib import nullcontext, suppress
  16. from contextvars import copy_context
  17. from gateway.config import Platform
  18. from gateway.media_repair import repair_explicit_computer_use_media_paths
  19. from gateway.platforms.base import BasePlatformAdapter
  20. from gateway.platforms.event import MessageEvent
  21. from gateway.session import (
  22. SessionSource, _session_key_namespace, build_channel_continuity_note,
  23. build_session_context,
  24. )
  25. from gateway.session_transcript import TranscriptReadError
  26. from gateway.turn_context import TurnContext
  27. from gateway.turn_lease import DEFAULT_LEASE_WAIT, TurnLeaseTimeoutError
  28. from hermes_constants import get_hermes_home_override
  29. from pathlib import Path
  30. from typing import Any, Callable, Dict, List, Optional, Tuple
  31. from utils import base_url_hostname
  32. if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
  33. from gateway.run import GatewayRunner
  34. from gateway.run_turn_runner import TurnRunner
  35. # Log-record parity with the origin module.
  36. logger = logging.getLogger("gateway.run")
  37. # ``_resolve_turn_agent_config``'s default tier: the runner's per-turn ``_service_tier``.
  38. _TIER_FROM_RUNNER = object()
  39. def _resolve_policy_agent_runtime(runner, policy, *, fallback=True) -> tuple[str, dict]:
  40. """``_resolve_session_agent_runtime`` for a session bound to a launch policy: the route's FROZEN
  41. config and launch credential, walking its own fallback chain on an AuthError unless
  42. ``fallback=False`` (a session control asking which route the session itself runs). Module-level
  43. so tests that bind the method onto bare namespaces still reach it."""
  44. from gateway.run import _runtime_agent_kwargs
  45. from gateway.session_policy import route_launch_key
  46. from gateway.session_worker_construct import route_runtime
  47. from gateway.session_authorities import active_authority
  48. authority = active_authority(runner)
  49. # A follow_profile_config session's route is the profile's current one, model and endpoint from one read.
  50. frozen, model = policy.effective(authority)
  51. provider = (frozen.get('model') or {}).get('provider')
  52. runtime, fallback_entry = route_runtime(policy, frozen, model, route_launch_key(authority, policy, frozen),
  53. fallback=fallback)
  54. if fallback_entry is not None:
  55. from hermes_cli.fallback_config import pre_agent_fallback_notice
  56. runner._pre_agent_fallback_notice = pre_agent_fallback_notice(
  57. provider or '', model or '',
  58. runtime.get('provider') or fallback_entry.get('provider') or 'unknown',
  59. fallback_entry.get('model') or 'default')
  60. return fallback_entry['model'], _runtime_agent_kwargs(runtime)
  61. return model, _runtime_agent_kwargs(runtime)
  62. def _recover_or_remember_model(runner, skey, model):
  63. """Last step of ``_resolve_session_agent_runtime``: remember a good model, recover an empty one."""
  64. # Final safety net: an empty model (transient config-cache miss) makes every API call 400 and
  65. # the session goes silent — reuse the last model resolved for this session, else process-wide.
  66. if not model:
  67. _lr_state = runner._peek_session_state(skey) if skey else None
  68. _lr_star = runner._peek_session_state("*")
  69. _recovered = (
  70. (_lr_state.conversation.last_resolved_model if _lr_state else "")
  71. or (_lr_star.conversation.last_resolved_model if _lr_star else "")
  72. )
  73. if _recovered:
  74. logger.warning(
  75. "Empty model resolved for session=%s — recovering "
  76. "last-known-good model %s (config read likely returned "
  77. "empty; see #35314)", skey or "", _recovered,
  78. )
  79. model = _recovered
  80. else:
  81. # Cache the good resolution for future recovery turns.
  82. if skey:
  83. runner._session_state(skey).conversation.last_resolved_model = model
  84. runner._session_state("*").conversation.last_resolved_model = model
  85. return model
  86. class GatewayTurnPrepareMixin:
  87. def _resolve_session_agent_runtime(
  88. self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None,
  89. user_config: Optional[dict] = None, fallback: bool = True,
  90. ) -> tuple[str, dict]:
  91. """Resolve model/runtime for a session.
  92. Priority (highest first): session ``/model`` → ``channel_overrides`` → global config/env
  93. (``_resolve_gateway_model(user_config)`` and default provider resolution). ``fallback=False``
  94. keeps a launch-policy session on its own route (see ``_resolve_policy_agent_runtime``)."""
  95. from gateway.run import (
  96. _credential_pool_for_provider, _get_channel_override, _resolve_gateway_model,
  97. _resolve_runtime_agent_kwargs, _resolve_runtime_agent_kwargs_for_provider,
  98. )
  99. from gateway.session_policy import policy_for_source
  100. # Every exit path starts clean: a stale notice must never attach to another session's next
  101. # turn (#74349). Set again below only when THIS resolution fell back.
  102. self._pre_agent_fallback_notice = None
  103. policy = policy_for_source(self, source) if source is not None else None
  104. if policy is not None:
  105. return _resolve_policy_agent_runtime(self, policy, fallback=fallback)
  106. skey = self._resolve_session_key_or_none(source, session_key)
  107. model = _resolve_gateway_model(user_config)
  108. if skey:
  109. self._rehydrate_session_model_override(skey)
  110. _override_state = self._peek_session_state(skey) if skey else None
  111. override = _override_state.conversation.model_override if _override_state else None
  112. if override:
  113. override_model = override.get("model", model)
  114. override_runtime = {
  115. k: override.get(k) for k in (
  116. "provider", "requested_provider", "api_key", "base_url", "api_mode",
  117. "max_tokens", "credential_pool", "request_overrides", "capabilities",
  118. )
  119. }
  120. override_runtime["capabilities"] = dict(override_runtime["capabilities"] or {})
  121. if override_runtime.get("api_key"):
  122. if override_runtime.get("credential_pool") is None:
  123. override_runtime["credential_pool"] = _credential_pool_for_provider(override.get("provider"))
  124. logger.debug(
  125. "Session model override (fast): session=%s config_model=%s -> override_model=%s provider=%s",
  126. skey or "", model, override_model, override_runtime.get("provider"),
  127. )
  128. return override_model, override_runtime
  129. # No api_key on the override (credentials failed to re-resolve at rehydrate): resolve them
  130. # for the override's own provider below, never layer it over the default provider's runtime.
  131. logger.debug(
  132. "Session model override (no api_key, fallback): session=%s config_model=%s override_model=%s",
  133. skey or "", model, override_model,
  134. )
  135. elif logger.isEnabledFor(logging.DEBUG):
  136. # The override_keys scan walks every session; only pay for it when DEBUG is on.
  137. logger.debug(
  138. "No session model override: session=%s config_model=%s override_keys=%s",
  139. skey or "", model,
  140. [
  141. _key for _key, _st in list(self._sessions_map().items())
  142. if _st.conversation.model_override is not None
  143. ][:5] or "[]",
  144. )
  145. runtime_kwargs, unavailable_override = None, None
  146. if override and override.get("provider"):
  147. try:
  148. runtime_kwargs = _resolve_runtime_agent_kwargs_for_provider(
  149. override["provider"], target_model=override.get("model") or None)
  150. except Exception as exc:
  151. # Layering the override on the default runtime sent its model to the default provider's
  152. # endpoint (openai-codex on the Nous URL). Run this turn on the whole default route and say
  153. # so; the persisted override is kept, so the next turn retries it.
  154. logger.warning("Session /model override provider %s unavailable: %s", override["provider"], exc,
  155. exc_info=True)
  156. unavailable_override, override = override, None
  157. if runtime_kwargs is None:
  158. runtime_kwargs = _resolve_runtime_agent_kwargs()
  159. # Private notice metadata must never reach an ``AIAgent(**runtime_kwargs)`` spread; the turn
  160. # runner surfaces it through the agent's one-shot fallback notice (#74349).
  161. self._pre_agent_fallback_notice = runtime_kwargs.pop("_fallback_notice", None)
  162. runtime_model = runtime_kwargs.pop("model", None)
  163. if runtime_model:
  164. logger.info("Runtime provider supplied explicit model override: %s -> %s", model, runtime_model)
  165. model = runtime_model
  166. if unavailable_override and not self._pre_agent_fallback_notice:
  167. from hermes_cli.fallback_config import pre_agent_fallback_notice
  168. self._pre_agent_fallback_notice = pre_agent_fallback_notice(
  169. unavailable_override["provider"], unavailable_override.get("model"), runtime_kwargs.get("provider"), model)
  170. cfg = getattr(self, "config", None) # getattr: bare object.__new__ test runners
  171. if cfg and source is not None:
  172. ch = _get_channel_override(
  173. cfg, source.platform, str(source.chat_id) if source.chat_id else "",
  174. thread_id=str(source.thread_id) if getattr(source, "thread_id", None) else None,
  175. parent_id=str(source.parent_chat_id) if getattr(source, "parent_chat_id", None) else None,
  176. )
  177. if ch:
  178. if ch.model:
  179. model = ch.model
  180. if ch.provider:
  181. runtime_kwargs = _resolve_runtime_agent_kwargs_for_provider(ch.provider, target_model=model or None)
  182. ch_runtime_model = runtime_kwargs.pop("model", None)
  183. # Adopt the provider's bundled model only when the override named none.
  184. if ch_runtime_model and not ch.model:
  185. model = ch_runtime_model
  186. if override and skey:
  187. model, runtime_kwargs = self._apply_session_model_override(skey, model, runtime_kwargs)
  188. # Provider resolved but no model.default (`hermes auth add` without `hermes model`): use the
  189. # provider's first catalog model.
  190. if not model and runtime_kwargs.get("provider"):
  191. with suppress(Exception):
  192. from hermes_cli.models import get_default_model_for_provider
  193. model = get_default_model_for_provider(runtime_kwargs["provider"])
  194. if model:
  195. logger.info(
  196. "No model configured — defaulting to %s for provider %s", model, runtime_kwargs["provider"],
  197. )
  198. return _recover_or_remember_model(self, skey, model), runtime_kwargs
  199. def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwargs: dict, *,
  200. tier: Any = _TIER_FROM_RUNNER) -> dict:
  201. """Effective model/runtime config for one turn. With `/fast` priority on, fast-mode
  202. ``request_overrides`` are deep-merged OVER the per-provider ones so both reach the model.
  203. ``tier`` defaults to the runner's per-turn ``_service_tier``; an off-turn builder (a cold
  204. ``/review``) passes the session's own instead of writing that shared slot."""
  205. from gateway.run import _deep_merge_request_overrides
  206. from agent.fast_mode import STATIC_TIERS
  207. from hermes_cli.models import resolve_fast_mode_overrides
  208. # Tests bind this method onto bare namespaces, so no class-level tables here.
  209. runtime = {
  210. k: runtime_kwargs.get(k) for k in (
  211. "api_key", "base_url", "provider", "requested_provider", "api_mode", "command", "args",
  212. "credential_pool", "max_tokens", "capabilities",
  213. )
  214. }
  215. runtime["args"] = list(runtime["args"] or [])
  216. runtime["capabilities"] = dict(runtime["capabilities"] or {})
  217. base_request_overrides = dict(runtime_kwargs.get("request_overrides") or {})
  218. route = {
  219. "model": model,
  220. "runtime": runtime,
  221. "signature": (
  222. model, runtime["provider"], runtime["requested_provider"], runtime["base_url"],
  223. runtime["api_mode"], runtime["command"], tuple(runtime["args"]),
  224. ),
  225. }
  226. if tier is _TIER_FROM_RUNNER:
  227. tier = getattr(self, "_service_tier", None)
  228. if tier not in STATIC_TIERS:
  229. # None / auto / cold: the bounded window is applied per request by agent.fast_mode.
  230. route["request_overrides"] = base_request_overrides
  231. return route
  232. try:
  233. overrides = resolve_fast_mode_overrides(
  234. route["model"], provider=runtime["provider"], base_url=runtime["base_url"], tier=tier,
  235. )
  236. except Exception:
  237. overrides = None
  238. # Fast-mode keys (service_tier / speed) are top-level and don't collide with extra_body.
  239. route["request_overrides"] = _deep_merge_request_overrides(base_request_overrides, overrides or {})
  240. return route
  241. def _sync_session_model_from_agent(self, session_id: str, agent: Any) -> None:
  242. """Persist the runtime model/provider a gateway turn actually used (provider fallback can
  243. switch them after the row was created). Runs in the ``run_sync`` executor thread, so it
  244. uses the sync ``SessionDB`` (``_db``), not the AsyncSessionDB forwarder."""
  245. if not session_id or agent is None or self._session_db is None:
  246. return
  247. model = getattr(agent, "model", None)
  248. if not model:
  249. return
  250. from hermes_cli.runtime_provider_custom import session_provider_identity
  251. runtime = {
  252. "provider": session_provider_identity(getattr(agent, "provider", None),
  253. getattr(agent, "requested_provider", None),
  254. getattr(agent, "base_url", None), model),
  255. "base_url": getattr(agent, "base_url", None),
  256. "api_mode": getattr(agent, "api_mode", None),
  257. }
  258. runtime["fallback_active"] = bool(getattr(agent, "_fallback_activated", False))
  259. runtime = {k: v for k, v in runtime.items() if v not in (None, "")}
  260. try:
  261. db = self._session_db._db
  262. row = db.get_session(session_id)
  263. if not row:
  264. return
  265. # Legacy backfill: canonical Bot Chats created BEFORE the follow_profile_config contract existed
  266. # carry no marker, yet they are still the plugin-owned forever-DM. The plugin's own identity
  267. # rule is "the profile's session titled exactly 'Bot Chat'" (UNIQUE(title) makes that an exact
  268. # registry, and pre-policy rows may be visible OR hidden), so mirror that rule here. Without
  269. # this, every Bot Chat that already exists in the field stays pinned to its stale stored
  270. # provider until the user deletes it — the exact live-report shape (#89497 / #94818).
  271. raw_config = row.get("model_config")
  272. config = {}
  273. with suppress(Exception):
  274. config = json.loads(raw_config) if raw_config else {}
  275. if not isinstance(config, dict):
  276. config = {}
  277. gateway_runtime = dict(config.get("gateway_runtime") or {})
  278. if row.get("model") == model and all(gateway_runtime.get(k) == v for k, v in runtime.items()):
  279. return
  280. config["gateway_runtime"] = runtime
  281. db.update_session_meta(session_id, json.dumps(config), model=model)
  282. except Exception:
  283. logger.debug("Failed to sync gateway session model metadata", exc_info=True)
  284. async def _hmwa_resolve_session(self, event, source):
  285. """Resolve ``source`` to its session entry (topic recovery, internal-route guards, Telegram
  286. topic-binding heal). Returns ``(source, session_entry, session_key)`` or ``None`` to drop
  287. the event."""
  288. # Topic-mode DMs: rewrite a stale/foreign thread_id to the user's last-active topic so a
  289. # cross-topic Reply doesn't fragment the conversation.
  290. event_metadata = getattr(event, "metadata", None) or {}
  291. expected_session_key = str(event_metadata.get("gateway_session_key") or "").strip()
  292. # Plain lobby recovery here even with ``dm_topic_auto_transfer`` on: the opt-in topic transfer
  293. # already ran at adapter ingress (``apply_root_dm_topic_transfer`` -> runner hook
  294. # ``_transfer_telegram_root_dm``), before session keying.
  295. recovered = (await asyncio.to_thread(self._recover_telegram_lobby_thread_id, source)
  296. if not expected_session_key else None)
  297. if recovered is not None:
  298. logger.info(
  299. "telegram topic recovery: chat=%s user=%s %r -> %s",
  300. source.chat_id, source.user_id, source.thread_id, recovered,
  301. )
  302. source = dataclasses.replace(source, thread_id=recovered)
  303. with suppress(Exception):
  304. event.source = source
  305. if expected_session_key:
  306. derived_session_key = self._session_key_for_source(source)
  307. if derived_session_key != expected_session_key:
  308. logger.warning(
  309. "Dropping internally routed event after route recovery: expected session=%s derived=%s",
  310. expected_session_key, derived_session_key,
  311. )
  312. return
  313. strict_session = bool(event_metadata.get("gateway_session_strict"))
  314. pinned_session_id = str(event_metadata.get("gateway_session_id") or "").strip()
  315. if strict_session:
  316. session_entry = await self.async_session_store.lookup_by_session_key(expected_session_key)
  317. if session_entry is None or not pinned_session_id or session_entry.session_id != pinned_session_id:
  318. logger.warning(
  319. "Dropping internally routed event: expected session id=%s is no longer current for key=%s",
  320. pinned_session_id or "missing", expected_session_key or "missing",
  321. )
  322. return
  323. else:
  324. # Internal wakes observe reset policy without counting as user activity, or periodic
  325. # notifications keep the routing key alive across every daily/idle boundary.
  326. session_entry = await self.async_session_store.get_or_create_session(
  327. source, touch_activity=not bool(getattr(event, "internal", False)),
  328. )
  329. session_key = session_entry.session_key
  330. claim = getattr(event, "_claimed_owner", None)
  331. if claim is not None:
  332. from gateway.session_authorities import active_authority
  333. authority, (owner, generation) = active_authority(self), claim
  334. # /new and /reset latch a Stop for the claimed generation before they move the route;
  335. # /resume, /branch and compression don't, so those turns run where the route points.
  336. stopped = generation in (authority.pending_stops.get(owner), authority.delivered_stops.get(owner))
  337. if stopped and session_entry.session_id not in await asyncio.to_thread(
  338. authority.db.get_compression_chain, owner): # preflight's continuation walk
  339. # It settles interrupted, as that Stop would have, and never runs on the fresh one.
  340. logger.warning("Dropping a stopped claimed turn: its route moved from %s to %s before it ran",
  341. owner, session_entry.session_id)
  342. from gateway.session_results import execution_result
  343. captured = execution_result.get()
  344. if captured is not None:
  345. captured['result'] = {'final_response': '', 'messages': [], 'interrupted': True}
  346. return
  347. if not strict_session and pinned_session_id:
  348. resolved_entry = await self._resolve_async_delegation_session(session_entry, pinned_session_id, event_metadata.get("gateway_proven_session_id") or "", event_metadata.get("gateway_completion_ids") or ())
  349. if resolved_entry is None:
  350. return
  351. session_entry = resolved_entry
  352. self._cache_session_source(session_key, source)
  353. if await asyncio.to_thread(self._is_telegram_topic_lane, source):
  354. session_entry = await self._hmwa_heal_telegram_topic_binding(source, session_entry, session_key)
  355. from gateway.run_heartbeat_acceptance import resolve_heartbeat_owner
  356. if not await resolve_heartbeat_owner(self, event, session_entry):
  357. return
  358. return source, session_entry, session_key
  359. async def _hmwa_heal_telegram_topic_binding(self, source, session_entry, session_key):
  360. """Follow the (chat_id, thread_id) topic binding — healed to its compression tip — or record
  361. a fresh one. Returns the (possibly switched) session entry."""
  362. binding = None
  363. try:
  364. if self._session_db:
  365. binding = await self._session_db.get_telegram_topic_binding(
  366. chat_id=str(source.chat_id), thread_id=str(source.thread_id),
  367. profile_name=self._telegram_topic_profile_name(source),
  368. )
  369. except Exception:
  370. logger.debug("Failed to read Telegram topic binding", exc_info=True)
  371. if not binding:
  372. try:
  373. await asyncio.to_thread(self._record_telegram_topic_binding, source, session_entry)
  374. except Exception:
  375. logger.debug("Failed to record Telegram topic binding", exc_info=True)
  376. return session_entry
  377. stored_session_id = str(binding.get("session_id") or "")
  378. bound_session_id = stored_session_id
  379. # A binding pointing at a pre-compression parent is walked forward to the tip so the next
  380. # message resumes the compressed child instead of reloading the oversized parent.
  381. # Returns the input unchanged when the session isn't a compression parent, so this is cheap and
  382. # safe. See #20470, #29712, #33414.
  383. if bound_session_id and self._session_db is not None:
  384. try:
  385. canonical_session_id = await self._session_db.get_compression_tip(bound_session_id)
  386. except Exception:
  387. logger.debug("compression-tip lookup failed for %s", bound_session_id, exc_info=True)
  388. canonical_session_id = bound_session_id
  389. if canonical_session_id and canonical_session_id != bound_session_id:
  390. bound_session_id = canonical_session_id
  391. if bound_session_id and bound_session_id != session_entry.session_id:
  392. # Route through SessionStore so the key → id mapping persists and the previous lane
  393. # session ends cleanly (in-place mutation split-brained the JSON index). The two DB
  394. # awaits above are a window for /new or /resume to move the route; the CAS on the
  395. # snapshot id lets that win instead of being clobbered by a stale binding.
  396. switched = await self.async_session_store.switch_session(
  397. session_key, bound_session_id, expected_session_id=session_entry.session_id,
  398. )
  399. if switched is not None:
  400. session_entry = switched
  401. if bound_session_id and bound_session_id != stored_session_id:
  402. # The stored binding pointed at a parent: rewrite it to the canonical descendant.
  403. await asyncio.to_thread(
  404. self._sync_telegram_topic_binding, source, session_entry, reason="compression-tip-walk",
  405. )
  406. return session_entry
  407. async def _hmwa_open_session(self, session_entry, session_key, source):
  408. """Consume auto-reset / fresh-reset flags and emit ``session:start`` for new sessions.
  409. Returns ``(_was_auto_reset, _is_new_session)``."""
  410. # Capture and immediately consume was_auto_reset so it does not re-fire on subsequent messages —
  411. # preventing the cleanup from wiping model/reasoning overrides set between turns (Closes #48031).
  412. _was_auto_reset = getattr(session_entry, "was_auto_reset", False)
  413. if _was_auto_reset:
  414. # Conversation boundary: the funnel clears every conversation-scoped dict; evict the cached
  415. # agent so context_compressor._previous_summary cannot leak into new summaries.
  416. # Treat auto-reset as a full conversation boundary — clear every conversation-scoped per-session
  417. # dict in one funnel call so the fresh session does not inherit the previous conversation's
  418. # model/reasoning overrides, a queued "/model switched" note, or a stale resolved-model cache
  419. # (#48031, #58403). See _CONVERSATION_SCOPED_STATE.
  420. self._clear_conversation_scope(session_key, reason="auto_reset")
  421. self._evict_cached_agent(session_key)
  422. session_entry.was_auto_reset = False
  423. _is_fresh_reset = getattr(session_entry, "is_fresh_reset", False)
  424. _is_new_session = session_entry.created_at == session_entry.updated_at or _was_auto_reset or _is_fresh_reset
  425. # Consume is_fresh_reset so it doesn't leak onto later messages in the same session.
  426. if _is_fresh_reset:
  427. # See #6508.
  428. session_entry.is_fresh_reset = False
  429. if _is_new_session:
  430. await self.hooks.emit("session:start", {
  431. "platform": source.platform.value if source.platform else "",
  432. "user_id": source.user_id,
  433. "session_id": session_entry.session_id,
  434. "session_key": session_key,
  435. })
  436. return _was_auto_reset, _is_new_session
  437. async def _hmwa_deliver_auto_reset_notice(self, session_entry, source, turn_sidecar_notes):
  438. """Stage the auto-reset sidecar note for the agent and notify the user (policy-gated)."""
  439. from gateway.run import _AUTO_RESET_CONTEXT_NOTES
  440. reset_reason = getattr(session_entry, 'auto_reset_reason', None) or 'suspended'
  441. context_note = _AUTO_RESET_CONTEXT_NOTES.get(reset_reason, _AUTO_RESET_CONTEXT_NOTES["suspended"])
  442. # Long-lived channels: point the agent at the prior same-channel session for session_search.
  443. try:
  444. # Returns None (appends nothing) for other platforms or when there's no prior activity to
  445. # recall. Deterministic — no extra API/DB calls (#36220).
  446. continuity_note = build_channel_continuity_note(session_entry, source)
  447. except Exception:
  448. continuity_note = None
  449. if continuity_note:
  450. context_note = context_note + "\n\n" + continuity_note
  451. turn_sidecar_notes.append(context_note)
  452. try:
  453. from gateway.bot_peer_notices import automatic_notices_allowed
  454. should_notify = reset_reason == "suspended" and automatic_notices_allowed(source)
  455. adapter = self._delivery_adapter_for(source) if should_notify else None
  456. if adapter:
  457. notice = t("gateway.session.auto_reset_notice")
  458. with suppress(Exception):
  459. session_info = await asyncio.to_thread(self._reset_notice_session_info, source)
  460. if session_info:
  461. notice = f"{notice}\n\n{session_info}"
  462. await adapter.send(source.chat_id, notice, metadata=self._thread_metadata_for_source(source))
  463. except Exception as e:
  464. logger.debug("Auto-reset notification failed (non-fatal): %s", e)
  465. # was_auto_reset was consumed in _hmwa_open_session; only the reason needs clearing.
  466. session_entry.auto_reset_reason = None
  467. def _hmwa_auto_load_skills(self, event, _auto, _quick_key, session_key):
  468. """Prepend topic/channel-bound skill payload(s) to ``event.text`` on a new session."""
  469. _skill_names = [_auto] if isinstance(_auto, str) else list(_auto)
  470. try:
  471. from agent.skill_commands import _load_skill_payload, _build_skill_message
  472. _combined_parts: list[str] = []
  473. _loaded_names: list[str] = []
  474. for _sname in _skill_names:
  475. _loaded = _load_skill_payload(_sname, task_id=_quick_key)
  476. if not _loaded:
  477. logger.warning("[Gateway] Auto-skill '%s' not found", _sname)
  478. continue
  479. _loaded_skill, _skill_dir, _display_name = _loaded
  480. _part = _build_skill_message(
  481. _loaded_skill, _skill_dir,
  482. f'[IMPORTANT: The "{_display_name}" skill is auto-loaded. '
  483. f"Follow its instructions for this session.]",
  484. )
  485. if _part:
  486. _combined_parts.append(_part)
  487. _loaded_names.append(_sname)
  488. if _combined_parts:
  489. _combined_parts.append(event.text) # user's original text after the payloads
  490. event.text = "\n\n".join(_combined_parts)
  491. logger.info("[Gateway] Auto-loaded skill(s) %s for session %s", _loaded_names, session_key)
  492. except Exception as e:
  493. logger.warning("[Gateway] Failed to auto-load skill(s) %s: %s", _skill_names, e)
  494. async def _hmwa_acquire_turn_lease(self, _quick_key, run_generation, session_entry, _session_env_tokens):
  495. """Serialize [load history → run → flush] per resolved SESSION_ID so another routing key on
  496. the same session waits for the prior flush. Fail-closed on timeout (outer dispatch returns
  497. a resend notice). Released in _handle_message's finally, granted per (routing key, run
  498. generation) so a stale unwind can't release a newer turn's."""
  499. from gateway.run import _float_env
  500. _lease_registry = getattr(self, "_turn_leases", None)
  501. if _lease_registry is None:
  502. return
  503. try:
  504. _lease_token = await _lease_registry.acquire(
  505. session_entry.session_id, owner_key=_quick_key, generation=run_generation,
  506. timeout=_float_env("HERMES_TURN_LEASE_TIMEOUT", DEFAULT_LEASE_WAIT),
  507. )
  508. except TurnLeaseTimeoutError:
  509. # The cleanup finally starts later; restore the tokens here or this exit leaks identity.
  510. self._clear_session_env(_session_env_tokens)
  511. raise
  512. if _lease_token is not None:
  513. self._session_state(_quick_key).turn.lease_tokens[run_generation] = _lease_token
  514. async def _hmwa_first_contact_notes(self, source, history, turn_sidecar_notes, internal=False):
  515. """First-ever-message onboarding note + one-time 'no home channel' prompt (both only when
  516. the session has no history). Delivered on the user message (sidecar), NOT the ephemeral
  517. system prompt: present-on-turn-1/absent-on-turn-2 was a guaranteed prompt diff + rebuild."""
  518. from gateway.run import _gateway_config_home, _home_target_env_var, _load_gateway_config
  519. if history or internal: # internal = plugin/system turn: no human made first contact
  520. return
  521. # LOCAL sessions (one-shot, chat -q, ACP, TUI attach) are the classic in-process CLI's
  522. # surfaces: it never appended this note, and mark_seen would rewrite config.yaml under
  523. # an ordinary launch that promises to leave the profile's files untouched.
  524. human_platform = bool(source.platform) and source.platform not in (Platform.LOCAL, Platform.WEBHOOK)
  525. if human_platform and source.chat_type == "dm" and not await self.async_session_store.has_any_sessions():
  526. # Same branch logic as the TUI (profile-build offer once when "ask", else plain intro);
  527. # first_contact_turn_note already falls back to the plain intro on error.
  528. from agent.onboarding import first_contact_turn_note
  529. note = first_contact_turn_note(
  530. _load_gateway_config(), _gateway_config_home() / "config.yaml",
  531. session_history_empty=True, install_has_prior_sessions=False,
  532. )
  533. if note:
  534. turn_sidecar_notes.append(note)
  535. # One-time prompt if no home channel is set (webhooks deliver to configured targets instead).
  536. if not human_platform:
  537. return
  538. platform_name = source.platform.value
  539. env_key = _home_target_env_var(platform_name)
  540. # Multiplex: the home channel may live only in the profile secret scope, not os.environ.
  541. home_env = ""
  542. if env_key:
  543. # A secondary with no home channel must not borrow the default profile's from
  544. # os.environ; only an UNSCOPED single-profile read may fall back to the process env.
  545. from agent.secret_scope import UnscopedSecretError, get_secret
  546. try:
  547. home_env = (get_secret(env_key) or "").strip()
  548. except UnscopedSecretError:
  549. home_env = (os.getenv(env_key) or "").strip()
  550. except Exception:
  551. logger.debug("Home-channel secret %s unreadable; treating it as unset", env_key, exc_info=True)
  552. home_env = ""
  553. # Also honor in-memory / yaml home_channel on this platform.
  554. with suppress(Exception):
  555. if not home_env and self.config.get_home_channel(source.platform):
  556. home_env = "set"
  557. # Secondary-profile platforms may only exist under that profile's config — re-read in scope.
  558. if not home_env:
  559. with suppress(Exception):
  560. from gateway.config import load_gateway_config as _lgc
  561. prof = (getattr(source, "profile", None) or "").strip()
  562. if prof and prof != "default" and _lgc().get_home_channel(source.platform):
  563. home_env = "set"
  564. if not home_env:
  565. # Slack routes every command through the parent `/hermes`; bare `/sethome` would fail.
  566. sethome_cmd = "/hermes sethome" if source.platform == Platform.SLACK else "/sethome"
  567. await self._deliver_platform_notice(
  568. source, t("gateway.notify.no_home_channel", platform=platform_name.title(), sethome_cmd=sethome_cmd),
  569. )
  570. def _hmwa_apply_message_timestamp(self, event, message_text):
  571. """Capture the platform event time as message metadata and keep the persisted transcript
  572. clean — strip any leading timestamp prefix and the Discord triggering-message note (a
  573. model instruction, not authored text) — regardless of the toggle; only the in-context
  574. RENDER is gated behind gateway.message_timestamps.enabled (default OFF)."""
  575. from gateway.run import _load_gateway_config, _message_timestamps_enabled
  576. from gateway.run_inbound import strip_discord_triggering_note
  577. persist_user_message = None
  578. persist_user_timestamp = None
  579. try:
  580. from hermes_time import get_timezone as _get_evt_tz
  581. from gateway.message_timestamps import (
  582. coerce_message_timestamp as _coerce_msg_ts,
  583. render_user_content_with_timestamp as _render_msg_ts,
  584. strip_leading_message_timestamps as _strip_msg_ts,
  585. )
  586. _evt_tz = _get_evt_tz()
  587. if message_text and isinstance(message_text, str):
  588. _clean_message_text, _embedded_ts = _strip_msg_ts(message_text, tz=_evt_tz)
  589. persist_user_message = strip_discord_triggering_note(event, _clean_message_text)
  590. _event_epoch = _coerce_msg_ts(getattr(event, "timestamp", None), tz=_evt_tz)
  591. persist_user_timestamp = _event_epoch if _event_epoch is not None else _embedded_ts
  592. if _message_timestamps_enabled(_load_gateway_config()):
  593. message_text = _render_msg_ts(_clean_message_text, persist_user_timestamp, tz=_evt_tz)
  594. else:
  595. # Toggle off: the model sees the clean message; timestamp stored for later opt-in.
  596. message_text = _clean_message_text
  597. except Exception as _ts_err:
  598. logger.debug("Message timestamp injection failed (non-fatal): %s", _ts_err)
  599. return message_text, persist_user_message, persist_user_timestamp
  600. @dataclasses.dataclass
  601. class _PreparedTurn:
  602. """Inputs to the agent run assembled by ``_hmwa_prepare_turn``."""
  603. history: Any
  604. context_prompt: str
  605. message_text: Any
  606. persist_user_message: Any
  607. persist_user_timestamp: Any
  608. persist_user_display_kind: Optional[str]
  609. persistence_session_id: Optional[str] = None
  610. persistence_owner: Optional[str] = None
  611. title_user_message: Optional[str] = None
  612. async def _hmwa_prepare_turn(self, event, source, session_entry, session_key, _quick_key, run_generation):
  613. """Everything between session resolution and the agent run: session open, task-local env,
  614. context prompt, sidecar notes, turn lease, transcript load + hygiene, inbound text. Returns
  615. ``(_PreparedTurn, env_tokens)``; a ``str`` first element is a reply to send instead of
  616. running (history unreadable, a local turn's refused ``@`` references); ``None`` drops the turn
  617. (inbound text rejected)."""
  618. from gateway.run import _load_gateway_config
  619. from tools.approval_yolo import restore_gateway_yolo
  620. _was_auto_reset, _is_new_session = await self._hmwa_open_session(session_entry, session_key, source)
  621. restore_gateway_yolo(session_key, session_entry.yolo) # None: TurnRunner seeds the launch policy
  622. context = build_session_context(source, self.config, session_entry)
  623. # Session context variables for tools (task-local, concurrency-safe)
  624. _session_env_tokens = self._set_session_env(context)
  625. # Self-injected turns (internal=True) persist with a DB-only display_kind: timeline notices, not user bubbles.
  626. from gateway.response_filters import display_kind_for_event
  627. persist_user_display_kind = display_kind_for_event(event)
  628. _redact_pii = False # privacy.redact_pii, re-read per message
  629. with suppress(Exception):
  630. _redact_pii = bool((_load_gateway_config().get("privacy") or {}).get("redact_pii", False))
  631. # The context prompt render is pinned per session, keyed by a hash of the renderer inputs, so
  632. # the system prompt cannot drift turn-over-turn; a miss (thread rename, /sethome) re-renders.
  633. # A LOCAL session (CLI one-shot, ACP, cron run-now, TUI attach) is the operator's own machine:
  634. # the messaging block ("Source: Local", connected platforms, cron delivery targets) was never
  635. # part of the classic in-process prompt, and the TUI gateway still builds that prompt in-process.
  636. # Rendering it here would make every surface hop of one durable session a system-prompt
  637. # (and prompt-cache) break.
  638. # Synthetic goal/heartbeat/resume turns carry no prompt inputs of their own, so they reuse the
  639. # pins exactly like internal events do (and still pass authorization, unlike internal ones).
  640. _reuse_pins = event.internal or getattr(event, "inherit_channel_pin", False)
  641. if source.platform != Platform.LOCAL and _reuse_pins and session_key:
  642. await self._rehydrate_prompt_pins(session_key, session_entry.session_id)
  643. context_prompt = "" if source.platform == Platform.LOCAL else \
  644. self._pinned_session_context_prompt(context, _redact_pii, session_key, internal=_reuse_pins)
  645. # Per-turn notes ride the user message via the api_content sidecar, NOT context_prompt
  646. # (appending to the ephemeral system prompt forced a full agent rebuild).
  647. turn_sidecar_notes: list[str] = []
  648. if _was_auto_reset:
  649. await self._hmwa_deliver_auto_reset_notice(session_entry, source, turn_sidecar_notes)
  650. # Keep human text separate from skills, sender metadata, and other model context.
  651. # With no text (e.g. voice-only), retain the existing enriched-message title fallback.
  652. title_user_message = event.text or None
  653. # Auto-load bound skill(s) only on NEW sessions; ongoing ones carry the content in history.
  654. _auto = getattr(event, "auto_skill", None)
  655. if _is_new_session and _auto:
  656. self._hmwa_auto_load_skills(event, _auto, _quick_key, session_key)
  657. await self._hmwa_acquire_turn_lease(_quick_key, run_generation, session_entry, _session_env_tokens)
  658. # A turn becomes durable recovery work only after it owns the per-session lease; marking
  659. # earlier would falsely recover a message that never began processing.
  660. await self._mark_durable_active_turn(event, session_entry.session_key)
  661. # An unreadable store is not an empty conversation: stop before the agent invents continuity
  662. # from []. Restore task-local context here (before the broad cleanup finally).
  663. try:
  664. history = await self.async_session_store.load_transcript(session_entry.session_id)
  665. history = await self._hmwa_run_session_hygiene(
  666. event, source, session_entry, session_key, history, _quick_key, run_generation,
  667. )
  668. except TranscriptReadError:
  669. self._clear_session_env(_session_env_tokens)
  670. from gateway.session_results import record_unexecuted_failure
  671. return record_unexecuted_failure(t("gateway.errors.history_unavailable")), _session_env_tokens
  672. await self._hmwa_first_contact_notes(source, history, turn_sidecar_notes, internal=event.internal)
  673. # Voice channel state rides the user message ONLY when changed (in the system prompt it
  674. # forced a rebuild + prompt-cache re-key per message).
  675. _vc_note = self._voice_channel_sidecar_note(event, source, session_key)
  676. if _vc_note:
  677. turn_sidecar_notes.append(_vc_note)
  678. # Auto-analyze user images so the model gets a description plus the local path.
  679. message_text = await self._prepare_profile_scoped_inbound_message_text(
  680. event=event, source=source, history=history, session_key=session_key,
  681. )
  682. if message_text is None:
  683. # A refused local turn answers with its refusal (session_context_refs.turn_references).
  684. return getattr(event, "_gateway_turn_refusal", None), _session_env_tokens
  685. message_text, persist_user_message, persist_user_timestamp = (
  686. self._hmwa_apply_message_timestamp(event, message_text)
  687. )
  688. # Stage the notes (one-shot; consumed in run_sync) AFTER the early-out so an aborted turn
  689. # cannot leak them into the next turn.
  690. if turn_sidecar_notes and session_key:
  691. self._set_pending_turn_sidecar_notes(session_key, turn_sidecar_notes)
  692. # Bind this run generation to the adapter so deferred post-delivery callbacks are released
  693. # by the run that registered them.
  694. self._bind_adapter_run_generation(self._delivery_adapter_for(source), session_key, run_generation)
  695. # Delivery IDs are only unique in their transport namespace. Keyless turns
  696. # need their own identity, even when another process writes to this session.
  697. import uuid
  698. namespace = [source.platform.value, source.profile, source.scope_id,
  699. source.chat_id, source.thread_id, str(event.message_id)]
  700. owner = (str(uuid.uuid5(uuid.NAMESPACE_URL, json.dumps(namespace)))
  701. if event.message_id else str(uuid.uuid4()))
  702. return self._PreparedTurn(
  703. history, context_prompt, message_text, persist_user_message, persist_user_timestamp,
  704. persist_user_display_kind, session_entry.session_id, owner,
  705. title_user_message=title_user_message,
  706. ), _session_env_tokens